@orion-js/dogs 4.6.8 → 4.6.9

package/dist/index.js

@@ -839,7 +839,7 @@ function scheduleJobsInternal(jobs, nPartitions) {
 
 // src/services/WorkerService.ts
 import { sleep } from "@orion-js/helpers";
-import { logger as logger4 } from "@orion-js/logger";
+import { logger as logger5 } from "@orion-js/logger";
 import { Inject as Inject3, Service as Service3 } from "@orion-js/services";
 
 // src/services/Executor.ts
@@ -1306,6 +1306,76 @@ __decorateElement(_init4, 5, "jobsHistoryRepo", _jobsHistoryRepo_dec, Executor);
 Executor = __decorateElement(_init4, 0, "Executor", _Executor_decorators, Executor);
 __runInitializers(_init4, 1, Executor);
 
+// src/services/jobFilters.ts
+import { logger as logger4 } from "@orion-js/logger";
+var DEFAULT_JOBS_FILTER_TIMEOUT_MS = 5e3;
+var warnedUnknownJobNames = /* @__PURE__ */ new Set();
+async function withTimeout(promise, timeoutMs, name) {
+  let timer;
+  const timeout = new Promise((_, reject) => {
+    timer = setTimeout(() => reject(new Error(`${name} timed out after ${timeoutMs}ms`)), timeoutMs);
+  });
+  try {
+    return await Promise.race([promise, timeout]);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+async function runCheck(options, name, run) {
+  if (options.pendingChecks.has(name)) {
+    throw new Error(`${name} is still running from a previous claim`);
+  }
+  const promise = Promise.resolve().then(run);
+  options.pendingChecks.set(name, promise);
+  void promise.then(
+    () => options.pendingChecks.delete(name),
+    () => options.pendingChecks.delete(name)
+  );
+  return await withTimeout(promise, options.timeoutMs, name);
+}
+function warnUnknownJobNames(jobs, filter) {
+  for (const jobName of [...filter.allow ?? [], ...filter.deny ?? []]) {
+    if (jobs[jobName] || warnedUnknownJobNames.has(jobName)) continue;
+    warnedUnknownJobNames.add(jobName);
+    logger4.warn(`getJobsFilter returned unknown job name "${jobName}"; it will be ignored.`);
+  }
+}
+async function filterByJobsFilter(options, getJobsFilter) {
+  try {
+    const filter = await runCheck(options, "getJobsFilter", getJobsFilter) ?? {};
+    warnUnknownJobNames(options.jobs, filter);
+    return options.jobNames.filter((jobName) => {
+      var _a;
+      if (filter.allow && !filter.allow.includes(jobName)) return false;
+      if ((_a = filter.deny) == null ? void 0 : _a.includes(jobName)) return false;
+      return true;
+    });
+  } catch (error) {
+    logger4.error("Error in getJobsFilter; no jobs will be taken on this claim.", { error });
+    return [];
+  }
+}
+async function canTakeJob(options, jobName) {
+  const canTakeJobs = options.jobs[jobName].canTakeJobs;
+  if (!canTakeJobs) return true;
+  try {
+    return await runCheck(options, `canTakeJobs of "${jobName}"`, canTakeJobs) === true;
+  } catch (error) {
+    logger4.error(`Error in canTakeJobs of job "${jobName}"; it will not be taken.`, { error });
+    return false;
+  }
+}
+async function getTakeableJobNames(inputOptions) {
+  const options = {
+    timeoutMs: DEFAULT_JOBS_FILTER_TIMEOUT_MS,
+    pendingChecks: /* @__PURE__ */ new Map(),
+    ...inputOptions
+  };
+  const jobNames = inputOptions.getJobsFilter ? await filterByJobsFilter(options, inputOptions.getJobsFilter) : options.jobNames;
+  const checks = await Promise.all(jobNames.map((jobName) => canTakeJob(options, jobName)));
+  return jobNames.filter((_, index) => checks[index]);
+}
+
 // src/services/WorkerService.ts
 var FILTERED_JOB_NAMES_ACQUISITION_HINT = "byJobName";
 var _executor_dec, _jobsRepo_dec3, _WorkerService_decorators, _init5;
@@ -1317,6 +1387,7 @@ var WorkerService = class {
     __publicField(this, "partitionReconcilerJitterMs", 30 * 1e3);
     __publicField(this, "partitionReconcilerBatchPauseMs", 250);
     __publicField(this, "partitionReconcilerBatchSize", 500);
+    __publicField(this, "jobsFilterTimeoutMs", DEFAULT_JOBS_FILTER_TIMEOUT_MS);
     __publicField(this, "jobsRepo", __runInitializers(_init5, 8, this)), __runInitializers(_init5, 11, this);
     __publicField(this, "executor", __runInitializers(_init5, 12, this)), __runInitializers(_init5, 15, this);
   }
@@ -1375,13 +1446,13 @@ var WorkerService = class {
     };
     this.reserveJobExecution(workersInstance, jobToRun.name);
     workersInstance.activeExecutions.set(jobToRun.executionId, execution);
-    logger4.debug(`Starting job execution "${jobToRun.executionId}"`, jobToRun);
+    logger5.debug(`Starting job execution "${jobToRun.executionId}"`, jobToRun);
     const executionPromise = this.executor.executeJob(config, jobToRun, () => {
       this.releaseExecutionCapacity(workersInstance, execution, "stale");
     });
     execution.promise = executionPromise;
     void executionPromise.catch((error) => {
-      logger4.error(`Unhandled error executing job "${jobToRun.name}"`, { error });
+      logger5.error(`Unhandled error executing job "${jobToRun.name}"`, { error });
     }).finally(() => {
       if (execution.status === "running") {
         this.releaseExecutionCapacity(workersInstance, execution, "finished");
@@ -1422,18 +1493,33 @@ var WorkerService = class {
         executionGuardRetryDelayMs: Math.max(config.pollInterval, 1e3)
       } : {}
     };
-    logger4.debug(`Running scheduler loop for jobs "${jobNames.join(", ")}"...`);
+    logger5.debug(`Running scheduler loop for jobs "${jobNames.join(", ")}"...`);
     let consecutivePartitionMisses = 0;
     while (workersInstance.running) {
       let didMiss = false;
       let allJobNamesAtCapacity = false;
       try {
         while (workersInstance.activeExecutions.size < config.workersCount) {
-          const availableJobNames = this.getAvailableJobNames(config, workersInstance, jobNames);
-          if (availableJobNames.length === 0) {
+          const jobNamesWithCapacity = this.getAvailableJobNames(config, workersInstance, jobNames);
+          if (jobNamesWithCapacity.length === 0) {
             allJobNamesAtCapacity = workersInstance.activeExecutions.size > 0;
             break;
           }
+          const availableJobNames = await Promise.race([
+            getTakeableJobNames({
+              jobs: config.jobs,
+              getJobsFilter: config.getJobsFilter,
+              jobNames: jobNamesWithCapacity,
+              timeoutMs: this.jobsFilterTimeoutMs,
+              pendingChecks: workersInstance.pendingFilterChecks
+            }),
+            workersInstance.stopSignal.then(() => [])
+          ]);
+          if (!workersInstance.running) break;
+          if (availableJobNames.length === 0) {
+            didMiss = true;
+            break;
+          }
           const jobAcquisitionHint = availableJobNames.length < jobNames.length ? FILTERED_JOB_NAMES_ACQUISITION_HINT : workersInstance.jobAcquisitionHint;
           const partition = this.getNextPartition(config, workersInstance);
           const jobToRun = await this.jobsRepo.getJobAndLock(
@@ -1444,7 +1530,7 @@ var WorkerService = class {
           );
           if (!jobToRun) {
             consecutivePartitionMisses++;
-            logger4.debug(`No job to run in partition ${partition}`);
+            logger5.debug(`No job to run in partition ${partition}`);
             if (consecutivePartitionMisses >= config.nPartitions) {
               consecutivePartitionMisses = 0;
               didMiss = true;
@@ -1457,7 +1543,7 @@ var WorkerService = class {
           if (!workersInstance.running) break;
         }
       } catch (error) {
-        logger4.error("Error in job scheduler.", { error });
+        logger5.error("Error in job scheduler.", { error });
         consecutivePartitionMisses = 0;
         didMiss = true;
       }
@@ -1509,7 +1595,7 @@ var WorkerService = class {
     } catch (error) {
       this.resetPendingJobAcquisitionHint(workersInstance);
       if (workersInstance.running) {
-        logger4.warn("Job acquisition hint probe failed; keeping the current hint.", { error });
+        logger5.warn("Job acquisition hint probe failed; keeping the current hint.", { error });
       }
       return;
     }
@@ -1522,12 +1608,12 @@ var WorkerService = class {
       workersInstance.hasCompletedInitialJobAcquisitionHintProbe = true;
       workersInstance.jobAcquisitionHint = winner;
       this.resetPendingJobAcquisitionHint(workersInstance);
-      logger4.info(`Selected job acquisition hint "${winner}" after startup probe.`);
+      logger5.info(`Selected job acquisition hint "${winner}" after startup probe.`);
       return;
     }
     if (winner === currentHint) {
       this.resetPendingJobAcquisitionHint(workersInstance);
-      logger4.debug(`Keeping job acquisition hint "${currentHint}" after probe.`);
+      logger5.debug(`Keeping job acquisition hint "${currentHint}" after probe.`);
       return;
     }
     if (workersInstance.pendingJobAcquisitionHint === winner) {
@@ -1537,14 +1623,14 @@ var WorkerService = class {
       workersInstance.pendingJobAcquisitionHintWins = 1;
     }
     if (workersInstance.pendingJobAcquisitionHintWins < 2) {
-      logger4.debug(
+      logger5.debug(
         `Keeping job acquisition hint "${currentHint}"; "${winner}" has one consecutive probe win.`
       );
       return;
     }
     workersInstance.jobAcquisitionHint = winner;
     this.resetPendingJobAcquisitionHint(workersInstance);
-    logger4.info(`Changed job acquisition hint from "${currentHint}" to "${winner}".`);
+    logger5.info(`Changed job acquisition hint from "${currentHint}" to "${winner}".`);
   }
   async waitForNextJobAcquisitionHintProbe(workersInstance) {
     await new Promise((resolve) => {
@@ -1581,7 +1667,7 @@ var WorkerService = class {
         batchSize: this.partitionReconcilerBatchSize
       });
     } catch (error) {
-      logger4.warn("Job partition reconciliation failed; it will be retried.", { error });
+      logger5.warn("Job partition reconciliation failed; it will be retried.", { error });
       return 0;
     }
   }
@@ -1621,6 +1707,7 @@ var WorkerService = class {
       activeExecutions: /* @__PURE__ */ new Map(),
       staleExecutions: /* @__PURE__ */ new Map(),
       runningJobsByName: /* @__PURE__ */ new Map(),
+      pendingFilterChecks: /* @__PURE__ */ new Map(),
       jobAcquisitionHint: INITIAL_JOB_ACQUISITION_HINT,
       jobAcquisitionHintProbeCycle: 0,
       hasCompletedInitialJobAcquisitionHintProbe: false,
@@ -1639,7 +1726,7 @@ var WorkerService = class {
         if (workersInstance.stopPromise) return workersInstance.stopPromise;
         workersInstance.stopPromise = (async () => {
           var _a, _b;
-          logger4.info("Stopping job scheduler...");
+          logger5.info("Stopping job scheduler...");
           workersInstance.running = false;
           workersInstance.signalStop();
           (_a = workersInstance.cancelJobAcquisitionHintProbeWait) == null ? void 0 : _a.call(workersInstance);
@@ -1664,17 +1751,17 @@ var WorkerService = class {
     const jobs = this.getJobs(config.jobs);
     await Promise.all(
       jobs.filter((job) => job.type === "recurrent").map(async (job) => {
-        logger4.debug(`Ensuring records for job "${job.name}"...`);
+        logger5.debug(`Ensuring records for job "${job.name}"...`);
         await this.jobsRepo.ensureJobRecord(job, getRandomPartition(config.nPartitions));
       })
     );
   }
   async runScheduler(config, workersInstance) {
-    logger4.debug("Will ensure maxTriesReached retention and TTL index");
-    void this.jobsRepo.ensureMaxTriesReachedRetention(config.maxTriesReachedRetentionMs).catch((error) => logger4.error("Error ensuring maxTriesReached retention", { error }));
-    logger4.debug("Will ensure records for recurrent jobs");
+    logger5.debug("Will ensure maxTriesReached retention and TTL index");
+    void this.jobsRepo.ensureMaxTriesReachedRetention(config.maxTriesReachedRetentionMs).catch((error) => logger5.error("Error ensuring maxTriesReached retention", { error }));
+    logger5.debug("Will ensure records for recurrent jobs");
     await this.ensureRecords(config);
-    logger4.info(`Starting job scheduler with concurrency ${config.workersCount}`);
+    logger5.info(`Starting job scheduler with concurrency ${config.workersCount}`);
     await this.runSchedulerLoop(config, workersInstance);
   }
   /**
@@ -1700,7 +1787,7 @@ var WorkerService = class {
     setNameToJobs(config.jobs, config.nPartitions);
     (_b = (_a = this.jobsRepo).configureJobPartitions) == null ? void 0 : _b.call(_a, this.getJobNames(config.jobs), config.nPartitions);
     const workersInstance = this.createWorkersInstanceDefinition(config);
-    logger4.debug("Starting job scheduler", config);
+    logger5.debug("Starting job scheduler", config);
     workersInstance.jobAcquisitionHintProbePromise = this.runJobAcquisitionHintProbeLoop(
       config,
       workersInstance
@@ -1717,7 +1804,7 @@ var WorkerService = class {
       workersInstance.signalStop();
       (_a2 = workersInstance.cancelJobAcquisitionHintProbeWait) == null ? void 0 : _a2.call(workersInstance);
       (_b2 = workersInstance.cancelPartitionReconcilerWait) == null ? void 0 : _b2.call(workersInstance);
-      logger4.error("Job scheduler stopped after an unexpected error.", { error });
+      logger5.error("Job scheduler stopped after an unexpected error.", { error });
     });
     return workersInstance;
   }