@orion-js/dogs 4.5.1 → 4.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/index.cjs CHANGED
@@ -115,6 +115,11 @@ var JobRecordSchema = (0, import_schema.schemaWithName)("JobRecord", {
115
115
  nextRunAt: { type: "date" },
116
116
  lastRunAt: { type: "date", optional: true },
117
117
  lockedUntil: { type: "date", optional: true },
118
+ /**
119
+ * Identifies the execution that currently owns the lock. Mutations from an execution must match
120
+ * this value so a stale execution cannot modify a newer claim.
121
+ */
122
+ lockId: { type: "string", optional: true },
118
123
  tries: { type: "number", optional: true },
119
124
  params: { type: "blackbox", optional: true },
120
125
  /**
@@ -182,6 +187,7 @@ var JobsRepo = class {
182
187
  __publicField(this, "jobs", __runInitializers(_init, 8, this)), __runInitializers(_init, 11, this);
183
188
  }
184
189
  async getJobAndLock(jobNames, lockTime) {
190
+ const executionId = (0, import_helpers.generateId)();
185
191
  const lockedUntil = new Date(Date.now() + lockTime);
186
192
  const job = await this.jobs.findOneAndUpdate(
187
193
  {
@@ -195,7 +201,7 @@ var JobsRepo = class {
195
201
  ]
196
202
  },
197
203
  {
198
- $set: { lockedUntil, lastRunAt: /* @__PURE__ */ new Date() },
204
+ $set: { lockId: executionId, lockedUntil, lastRunAt: /* @__PURE__ */ new Date() },
199
205
  $inc: { tries: 1 }
200
206
  },
201
207
  {
@@ -216,7 +222,8 @@ var JobsRepo = class {
216
222
  }
217
223
  return {
218
224
  jobId: job._id,
219
- executionId: (0, import_helpers.generateId)(),
225
+ executionId,
226
+ lockId: executionId,
220
227
  name: job.jobName,
221
228
  params: job.params,
222
229
  type: job.type,
@@ -227,8 +234,11 @@ var JobsRepo = class {
227
234
  wasStale
228
235
  };
229
236
  }
230
- async setJobRecordPriority(jobId, priority) {
231
- await this.jobs.updateOne(jobId, { $set: { priority } });
237
+ async setJobRecordPriority(jobId, priority, lockId) {
238
+ const result = await this.jobs.updateOne(lockId ? { _id: jobId, lockId } : jobId, {
239
+ $set: { priority }
240
+ });
241
+ return result.matchedCount > 0;
232
242
  }
233
243
  async scheduleNextRun(options) {
234
244
  const updator = {
@@ -237,22 +247,31 @@ var JobsRepo = class {
237
247
  priority: options.priority,
238
248
  ...options.resetTries ? { tries: 0 } : {}
239
249
  },
240
- $unset: { lockedUntil: "" }
250
+ $unset: { lockId: "", lockedUntil: "" }
241
251
  };
242
- await this.jobs.updateOne(options.jobId, updator);
252
+ const result = await this.jobs.updateOne(
253
+ options.lockId ? { _id: options.jobId, lockId: options.lockId } : options.jobId,
254
+ updator
255
+ );
256
+ return result.matchedCount > 0;
243
257
  }
244
- async deleteEventJob(jobId) {
245
- await this.jobs.deleteOne({ _id: jobId, type: "event" });
258
+ async deleteEventJob(jobId, lockId) {
259
+ const result = await this.jobs.deleteOne({
260
+ _id: jobId,
261
+ type: "event",
262
+ ...lockId ? { lockId } : {}
263
+ });
264
+ return result.deletedCount > 0;
246
265
  }
247
266
  /**
248
267
  * Marks an event job as having reached its maximum tries limit.
249
268
  * The job won't be picked up for execution and MongoDB will delete it after
250
269
  * the configured retention period.
251
270
  */
252
- async markJobAsMaxTriesReached(jobId, retentionMs = DEFAULT_MAX_TRIES_REACHED_RETENTION_MS) {
271
+ async markJobAsMaxTriesReached(jobId, retentionMs = DEFAULT_MAX_TRIES_REACHED_RETENTION_MS, lockId) {
253
272
  const maxTriesReachedAt = /* @__PURE__ */ new Date();
254
- await this.jobs.updateOne(
255
- { _id: jobId, type: "event" },
273
+ const result = await this.jobs.updateOne(
274
+ { _id: jobId, type: "event", ...lockId ? { lockId } : {} },
256
275
  {
257
276
  $set: {
258
277
  status: "maxTriesReached",
@@ -260,11 +279,13 @@ var JobsRepo = class {
260
279
  ...retentionMs === null ? {} : { expiresAt: new Date(maxTriesReachedAt.getTime() + retentionMs) }
261
280
  },
262
281
  $unset: {
282
+ lockId: "",
263
283
  lockedUntil: "",
264
284
  ...retentionMs === null ? { expiresAt: "" } : {}
265
285
  }
266
286
  }
267
287
  );
288
+ return result.matchedCount > 0;
268
289
  }
269
290
  /**
270
291
  * Applies the current retention configuration to existing maxTriesReached
@@ -293,23 +314,22 @@ var JobsRepo = class {
293
314
  }
294
315
  ]);
295
316
  }
296
- async extendLockTime(jobId, extraTime) {
297
- await this.updateLockTime(jobId, extraTime);
317
+ async extendLockTime(jobId, extraTime, lockId) {
318
+ return this.updateLockTime(jobId, extraTime, lockId);
298
319
  }
299
320
  /**
300
321
  * Updates the lock time for a job to the specified duration from now.
301
322
  * Can be used to both extend or shorten the lock time.
302
323
  */
303
- async updateLockTime(jobId, lockDuration) {
324
+ async updateLockTime(jobId, lockDuration, lockId) {
304
325
  const lockedUntil = new Date(Date.now() + lockDuration);
305
- await this.jobs.updateOne(
306
- {
307
- _id: jobId
308
- },
326
+ const result = await this.jobs.updateOne(
327
+ { _id: jobId, ...lockId ? { lockId } : {} },
309
328
  {
310
329
  $set: { lockedUntil }
311
330
  }
312
331
  );
332
+ return result.matchedCount > 0;
313
333
  }
314
334
  async unlockAllJobs() {
315
335
  const result = await this.jobs.updateMany(
@@ -317,7 +337,7 @@ var JobsRepo = class {
317
337
  lockedUntil: { $exists: true }
318
338
  },
319
339
  {
320
- $unset: { lockedUntil: "" }
340
+ $unset: { lockId: "", lockedUntil: "" }
321
341
  }
322
342
  );
323
343
  return result.modifiedCount;
@@ -623,11 +643,15 @@ var Executor = class {
623
643
  getEffectiveLockTime(job, jobToRun) {
624
644
  return job.lockTime ?? jobToRun.lockTime;
625
645
  }
626
- getContext(job, jobToRun, onStale) {
646
+ getContext(job, jobToRun, onStale, isExecutionActive) {
627
647
  var _a;
628
648
  const effectiveLockTime = this.getEffectiveLockTime(job, jobToRun);
629
649
  let staleTimeout = setTimeout(() => onStale(), effectiveLockTime);
630
650
  (_a = staleTimeout.unref) == null ? void 0 : _a.call(staleTimeout);
651
+ const contextLogger = import_logger3.logger.addMetadata({
652
+ jobName: jobToRun.name,
653
+ jobId: jobToRun.jobId
654
+ });
631
655
  return {
632
656
  definition: job,
633
657
  record: jobToRun,
@@ -635,15 +659,26 @@ var Executor = class {
635
659
  clearStaleTimeout: () => clearTimeout(staleTimeout),
636
660
  extendLockTime: async (extraTime) => {
637
661
  var _a2;
662
+ if (!isExecutionActive()) {
663
+ contextLogger.warn(`Will not extend lock for stale execution "${jobToRun.executionId}"`);
664
+ return;
665
+ }
638
666
  clearTimeout(staleTimeout);
667
+ let didExtend = false;
668
+ try {
669
+ didExtend = await this.jobsRepo.extendLockTime(jobToRun.jobId, extraTime, jobToRun.lockId);
670
+ } catch (error) {
671
+ onStale();
672
+ throw error;
673
+ }
674
+ if (!didExtend) {
675
+ onStale();
676
+ return;
677
+ }
639
678
  staleTimeout = setTimeout(() => onStale(), extraTime);
640
679
  (_a2 = staleTimeout.unref) == null ? void 0 : _a2.call(staleTimeout);
641
- await this.jobsRepo.extendLockTime(jobToRun.jobId, extraTime);
642
680
  },
643
- logger: import_logger3.logger.addMetadata({
644
- jobName: jobToRun.name,
645
- jobId: jobToRun.jobId
646
- })
681
+ logger: contextLogger
647
682
  };
648
683
  }
649
684
  getJobDefinition(jobToRun, jobs) {
@@ -675,8 +710,16 @@ var Executor = class {
675
710
  jobLogger.warn(
676
711
  `Job "${jobToRun.name}" has exceeded max tries (${jobToRun.tries}). Marking as maxTriesReached.`
677
712
  );
678
- await this.jobsRepo.markJobAsMaxTriesReached(jobToRun.jobId, retentionMs);
679
- if (!onMaxTriesReached) return;
713
+ const didMark = await this.jobsRepo.markJobAsMaxTriesReached(
714
+ jobToRun.jobId,
715
+ retentionMs,
716
+ jobToRun.lockId
717
+ );
718
+ if (!didMark) {
719
+ jobLogger.warn(`Could not mark job "${jobToRun.name}" because its lock is no longer owned`);
720
+ return false;
721
+ }
722
+ if (!onMaxTriesReached) return true;
680
723
  try {
681
724
  await onMaxTriesReached(jobToRun);
682
725
  } catch (callbackError) {
@@ -684,30 +727,33 @@ var Executor = class {
684
727
  error: callbackError
685
728
  });
686
729
  }
730
+ return true;
687
731
  }
688
732
  async onError(error, job, jobToRun, context) {
689
733
  const scheduleRecurrent = async () => {
690
734
  if (job.type === "recurrent") {
691
- await this.jobsRepo.scheduleNextRun({
735
+ return this.jobsRepo.scheduleNextRun({
692
736
  jobId: jobToRun.jobId,
693
737
  nextRunAt: getNextRunDate(job),
694
738
  resetTries: true,
695
- priority: job.priority
739
+ priority: job.priority,
740
+ lockId: jobToRun.lockId
696
741
  });
697
742
  }
743
+ return true;
698
744
  };
699
745
  const handleRetry = async (nextRunAt) => {
700
- await this.jobsRepo.scheduleNextRun({
746
+ return this.jobsRepo.scheduleNextRun({
701
747
  jobId: jobToRun.jobId,
702
748
  nextRunAt,
703
749
  resetTries: false,
704
- priority: job.type === "recurrent" ? job.priority : jobToRun.priority
750
+ priority: job.type === "recurrent" ? job.priority : jobToRun.priority,
751
+ lockId: jobToRun.lockId
705
752
  });
706
753
  };
707
754
  if (!job.onError) {
708
755
  context.logger.error(`Error executing job "${jobToRun.name}"`, { error });
709
- await scheduleRecurrent();
710
- return;
756
+ return scheduleRecurrent();
711
757
  }
712
758
  context.logger.info(`Error executing job "${jobToRun.name}"`, { error });
713
759
  const result = await job.onError(
@@ -716,12 +762,12 @@ var Executor = class {
716
762
  context
717
763
  );
718
764
  if (result.action === "dismiss") {
719
- await scheduleRecurrent();
720
- return;
765
+ return scheduleRecurrent();
721
766
  }
722
767
  if (result.action === "retry") {
723
- await handleRetry(getNextRunDate(result));
768
+ return handleRetry(getNextRunDate(result));
724
769
  }
770
+ return true;
725
771
  }
726
772
  async saveExecution(options) {
727
773
  const { startedAt, status, errorMessage, result, job, jobToRun } = options;
@@ -751,19 +797,21 @@ var Executor = class {
751
797
  async afterExecutionSuccess(job, jobToRun, context) {
752
798
  if (job.type === "recurrent") {
753
799
  context.logger.debug(`Scheduling next run for recurrent job "${jobToRun.name}"`);
754
- await this.jobsRepo.scheduleNextRun({
800
+ return this.jobsRepo.scheduleNextRun({
755
801
  jobId: jobToRun.jobId,
756
802
  nextRunAt: getNextRunDate(job),
757
803
  resetTries: true,
758
- priority: job.priority
804
+ priority: job.priority,
805
+ lockId: jobToRun.lockId
759
806
  });
760
807
  }
761
808
  if (job.type === "event") {
762
809
  context.logger.debug(`Removing event job after success "${jobToRun.name}"`);
763
- await this.jobsRepo.deleteEventJob(jobToRun.jobId);
810
+ return this.jobsRepo.deleteEventJob(jobToRun.jobId, jobToRun.lockId);
764
811
  }
812
+ return false;
765
813
  }
766
- async executeJob(config, jobToRun, respawnWorker) {
814
+ async executeJob(config, jobToRun, onExecutionStale) {
767
815
  const job = this.getJobDefinition(jobToRun, config.jobs);
768
816
  if (!job) return;
769
817
  if (job.type === "event") {
@@ -779,33 +827,64 @@ var Executor = class {
779
827
  }
780
828
  const effectiveLockTime = this.getEffectiveLockTime(job, jobToRun);
781
829
  if (effectiveLockTime !== jobToRun.lockTime) {
782
- await this.jobsRepo.updateLockTime(jobToRun.jobId, effectiveLockTime);
830
+ const didUpdateLock = await this.jobsRepo.updateLockTime(
831
+ jobToRun.jobId,
832
+ effectiveLockTime,
833
+ jobToRun.lockId
834
+ );
835
+ if (!didUpdateLock) {
836
+ import_logger3.logger.warn(`Will not execute job "${jobToRun.name}" because its lock is no longer owned`);
837
+ return;
838
+ }
783
839
  }
784
840
  const tracer = import_api.trace.getTracer("orionjs.dogs", "1.0");
785
841
  await tracer.startActiveSpan(`job.${jobToRun.name}.${jobToRun.executionId}`, async (span) => {
786
842
  try {
787
843
  const startedAt = /* @__PURE__ */ new Date();
788
- const onStale = async () => {
789
- if (job.onStale) {
790
- context.logger.info(`Job "${jobToRun.name}" is stale`);
791
- job.onStale(jobToRun.params, context);
792
- } else {
793
- context.logger.error(`Job "${jobToRun.name}" is stale`);
794
- }
795
- await this.jobsRepo.setJobRecordPriority(jobToRun.jobId, 0);
796
- void respawnWorker();
797
- void this.saveExecution({
798
- startedAt,
799
- status: "stale",
800
- result: null,
801
- errorMessage: null,
802
- job,
803
- jobToRun
804
- }).catch((error) => {
805
- context.logger.error("Error saving stale execution history", { error });
844
+ let executionStatus = "running";
845
+ let staleHandlingPromise;
846
+ let context;
847
+ const markAsStale = () => {
848
+ if (executionStatus === "stale") return staleHandlingPromise;
849
+ executionStatus = "stale";
850
+ context.clearStaleTimeout();
851
+ staleHandlingPromise = (async () => {
852
+ context.logger.warn(`Job "${jobToRun.name}" is stale`);
853
+ if (job.onStale) {
854
+ void Promise.resolve(job.onStale(jobToRun.params, context)).catch((error) => {
855
+ context.logger.error(`Error in onStale callback for job "${jobToRun.name}"`, {
856
+ error
857
+ });
858
+ });
859
+ }
860
+ try {
861
+ await this.jobsRepo.setJobRecordPriority(jobToRun.jobId, 0, jobToRun.lockId);
862
+ } catch (error) {
863
+ context.logger.error(`Error lowering priority for stale job "${jobToRun.name}"`, {
864
+ error
865
+ });
866
+ } finally {
867
+ onExecutionStale();
868
+ }
869
+ await this.saveExecution({
870
+ startedAt,
871
+ status: "stale",
872
+ result: null,
873
+ errorMessage: null,
874
+ job,
875
+ jobToRun
876
+ });
877
+ })().catch((error) => {
878
+ context.logger.error("Error handling stale execution", { error });
806
879
  });
880
+ return staleHandlingPromise;
807
881
  };
808
- const context = this.getContext(job, jobToRun, onStale);
882
+ context = this.getContext(
883
+ job,
884
+ jobToRun,
885
+ () => void markAsStale(),
886
+ () => executionStatus === "running"
887
+ );
809
888
  const extraContext = {
810
889
  controllerType: "job",
811
890
  jobName: jobToRun.name,
@@ -821,6 +900,18 @@ var Executor = class {
821
900
  });
822
901
  const result = await job.resolve(jobToRun.params, context);
823
902
  context.clearStaleTimeout();
903
+ if (executionStatus === "stale") {
904
+ await staleHandlingPromise;
905
+ context.logger.warn(
906
+ `Ignoring late success from stale execution "${jobToRun.executionId}"`
907
+ );
908
+ return;
909
+ }
910
+ const stillOwnsJob = await this.afterExecutionSuccess(job, jobToRun, context);
911
+ if (!stillOwnsJob) {
912
+ await markAsStale();
913
+ return;
914
+ }
824
915
  void this.saveExecution({
825
916
  startedAt,
826
917
  status: "success",
@@ -831,9 +922,21 @@ var Executor = class {
831
922
  }).catch((error) => {
832
923
  context.logger.error("Error saving successful execution history", { error });
833
924
  });
834
- await this.afterExecutionSuccess(job, jobToRun, context);
835
925
  } catch (error) {
836
926
  context.clearStaleTimeout();
927
+ if (executionStatus === "stale") {
928
+ await staleHandlingPromise;
929
+ context.logger.warn(
930
+ `Ignoring late error from stale execution "${jobToRun.executionId}"`,
931
+ { error }
932
+ );
933
+ return;
934
+ }
935
+ const stillOwnsJob = await this.onError(error, job, jobToRun, context);
936
+ if (!stillOwnsJob) {
937
+ await markAsStale();
938
+ return;
939
+ }
837
940
  void this.saveExecution({
838
941
  startedAt,
839
942
  status: "error",
@@ -844,7 +947,6 @@ var Executor = class {
844
947
  }).catch((saveError) => {
845
948
  context.logger.error("Error saving failed execution history", { error: saveError });
846
949
  });
847
- await this.onError(error, job, jobToRun, context);
848
950
  }
849
951
  });
850
952
  } catch (error) {
@@ -904,87 +1006,131 @@ var WorkerService = class {
904
1006
  }
905
1007
  workersInstance.runningJobsByName.set(jobName, currentExecutions - 1);
906
1008
  }
907
- async withJobAcquisitionLock(workersInstance, callback) {
908
- const previousLock = workersInstance.jobAcquisitionLock;
909
- let releaseLock;
910
- workersInstance.jobAcquisitionLock = new Promise((resolve) => {
911
- releaseLock = resolve;
912
- });
913
- await previousLock;
914
- try {
915
- return await callback();
916
- } finally {
917
- releaseLock();
1009
+ releaseExecutionCapacity(workersInstance, execution, nextStatus) {
1010
+ if (execution.status !== "running") return;
1011
+ execution.status = nextStatus;
1012
+ workersInstance.activeExecutions.delete(execution.job.executionId);
1013
+ this.releaseJobExecution(workersInstance, execution.job.name);
1014
+ execution.releaseCapacity();
1015
+ if (nextStatus === "stale") {
1016
+ workersInstance.staleExecutions.set(execution.job.executionId, execution);
918
1017
  }
919
1018
  }
920
- async getJobAndReserveExecution(config, workersInstance, jobNames) {
921
- return this.withJobAcquisitionLock(workersInstance, async () => {
922
- const availableJobNames = this.getAvailableJobNames(config, workersInstance, jobNames);
923
- if (availableJobNames.length === 0) return;
924
- const jobToRun = await this.jobsRepo.getJobAndLock(availableJobNames, config.defaultLockTime);
925
- if (!jobToRun) return;
926
- this.reserveJobExecution(workersInstance, jobToRun.name);
927
- return jobToRun;
1019
+ startExecution(config, workersInstance, jobToRun) {
1020
+ let releaseCapacity;
1021
+ const capacityPromise = new Promise((resolve) => {
1022
+ releaseCapacity = resolve;
1023
+ });
1024
+ const execution = {
1025
+ job: jobToRun,
1026
+ promise: Promise.resolve(),
1027
+ capacityPromise,
1028
+ releaseCapacity,
1029
+ status: "running"
1030
+ };
1031
+ this.reserveJobExecution(workersInstance, jobToRun.name);
1032
+ workersInstance.activeExecutions.set(jobToRun.executionId, execution);
1033
+ import_logger4.logger.debug(`Starting job execution "${jobToRun.executionId}"`, jobToRun);
1034
+ const executionPromise = this.executor.executeJob(config, jobToRun, () => {
1035
+ this.releaseExecutionCapacity(workersInstance, execution, "stale");
1036
+ });
1037
+ execution.promise = executionPromise;
1038
+ void executionPromise.catch((error) => {
1039
+ import_logger4.logger.error(`Unhandled error executing job "${jobToRun.name}"`, { error });
1040
+ }).finally(() => {
1041
+ if (execution.status === "running") {
1042
+ this.releaseExecutionCapacity(workersInstance, execution, "finished");
1043
+ } else if (execution.status === "stale") {
1044
+ workersInstance.staleExecutions.delete(jobToRun.executionId);
1045
+ }
928
1046
  });
929
1047
  }
930
- async runWorkerLoop(config, workersInstance, workerInstance, jobNames, executeConfig) {
931
- const jobToRun = await this.getJobAndReserveExecution(config, workersInstance, jobNames);
932
- if (!jobToRun) {
933
- import_logger4.logger.debug("No job to run");
934
- return false;
935
- }
936
- import_logger4.logger.debug(`Got job [w${workerInstance.workerIndex}] to run:`, jobToRun);
937
- try {
938
- await this.executor.executeJob(executeConfig, jobToRun, workerInstance.respawn);
939
- } finally {
940
- this.releaseJobExecution(workersInstance, jobToRun.name);
941
- }
942
- return true;
1048
+ async waitForPollOrStop(config, workersInstance) {
1049
+ await Promise.race([(0, import_helpers2.sleep)(config.pollInterval), workersInstance.stopSignal]);
943
1050
  }
944
- async startWorker(config, workersInstance, workerInstance) {
945
- const names = this.getJobNames(config.jobs);
946
- import_logger4.logger.debug(
947
- `Running worker loop [w${workerInstance.workerIndex}] for jobs "${names.join(", ")}"...`
1051
+ async waitForCapacityOrStop(workersInstance) {
1052
+ const capacityPromises = Array.from(
1053
+ workersInstance.activeExecutions.values(),
1054
+ (execution) => execution.capacityPromise
948
1055
  );
1056
+ if (capacityPromises.length === 0) return;
1057
+ await Promise.race([...capacityPromises, workersInstance.stopSignal]);
1058
+ }
1059
+ async runSchedulerLoop(config, workersInstance) {
1060
+ const jobNames = this.getJobNames(config.jobs);
949
1061
  const executeConfig = {
950
1062
  jobs: config.jobs,
951
1063
  maxTries: config.maxTries,
952
1064
  maxTriesReachedRetentionMs: config.maxTriesReachedRetentionMs,
953
1065
  onMaxTriesReached: config.onMaxTriesReached
954
1066
  };
955
- while (true) {
956
- if (!workerInstance.running) {
957
- import_logger4.logger.info(`Got signal to stop. Stopping worker [w${workerInstance.workerIndex}]...`);
958
- return;
959
- }
1067
+ import_logger4.logger.debug(`Running scheduler loop for jobs "${jobNames.join(", ")}"...`);
1068
+ while (workersInstance.running) {
1069
+ let didMiss = false;
1070
+ let allJobNamesAtCapacity = false;
960
1071
  try {
961
- const didRun = await this.runWorkerLoop(
962
- config,
963
- workersInstance,
964
- workerInstance,
965
- names,
966
- executeConfig
967
- );
968
- if (!didRun) await (0, import_helpers2.sleep)(config.pollInterval);
969
- if (didRun) await (0, import_helpers2.sleep)(config.cooldownPeriod);
1072
+ while (workersInstance.activeExecutions.size < config.workersCount) {
1073
+ const availableJobNames = this.getAvailableJobNames(config, workersInstance, jobNames);
1074
+ if (availableJobNames.length === 0) {
1075
+ allJobNamesAtCapacity = workersInstance.activeExecutions.size > 0;
1076
+ break;
1077
+ }
1078
+ const jobToRun = await this.jobsRepo.getJobAndLock(
1079
+ availableJobNames,
1080
+ config.defaultLockTime
1081
+ );
1082
+ if (!jobToRun) {
1083
+ import_logger4.logger.debug("No job to run");
1084
+ didMiss = true;
1085
+ break;
1086
+ }
1087
+ this.startExecution(executeConfig, workersInstance, jobToRun);
1088
+ if (!workersInstance.running) break;
1089
+ }
970
1090
  } catch (error) {
971
- import_logger4.logger.error("Error in job runner.", { error });
972
- await (0, import_helpers2.sleep)(config.pollInterval);
1091
+ import_logger4.logger.error("Error in job scheduler.", { error });
1092
+ didMiss = true;
1093
+ }
1094
+ if (!workersInstance.running) return;
1095
+ const concurrencyIsFull = workersInstance.activeExecutions.size >= config.workersCount && workersInstance.activeExecutions.size > 0;
1096
+ if (concurrencyIsFull || allJobNamesAtCapacity) {
1097
+ await this.waitForCapacityOrStop(workersInstance);
1098
+ } else if (didMiss || workersInstance.activeExecutions.size === 0) {
1099
+ await this.waitForPollOrStop(config, workersInstance);
973
1100
  }
974
1101
  }
975
1102
  }
976
1103
  createWorkersInstanceDefinition(config) {
1104
+ let signalStop;
1105
+ const stopSignal = new Promise((resolve) => {
1106
+ signalStop = resolve;
1107
+ });
977
1108
  const workersInstance = {
978
1109
  running: true,
979
1110
  workersCount: config.workersCount,
980
- workers: [],
1111
+ activeExecutions: /* @__PURE__ */ new Map(),
1112
+ staleExecutions: /* @__PURE__ */ new Map(),
981
1113
  runningJobsByName: /* @__PURE__ */ new Map(),
982
- jobAcquisitionLock: Promise.resolve(),
1114
+ schedulerPromise: Promise.resolve(),
1115
+ stopSignal,
1116
+ signalStop,
1117
+ get runningExecutions() {
1118
+ return this.activeExecutions.size;
1119
+ },
983
1120
  stop: async () => {
984
- import_logger4.logger.info("Stopping workers...");
985
- workersInstance.running = false;
986
- const stopingPromises = workersInstance.workers.map((worker) => worker.stop());
987
- await Promise.all(stopingPromises);
1121
+ if (workersInstance.stopPromise) return workersInstance.stopPromise;
1122
+ workersInstance.stopPromise = (async () => {
1123
+ import_logger4.logger.info("Stopping job scheduler...");
1124
+ workersInstance.running = false;
1125
+ workersInstance.signalStop();
1126
+ await workersInstance.schedulerPromise;
1127
+ const activeCapacity = Array.from(
1128
+ workersInstance.activeExecutions.values(),
1129
+ (execution) => execution.capacityPromise
1130
+ );
1131
+ await Promise.all(activeCapacity);
1132
+ })();
1133
+ return workersInstance.stopPromise;
988
1134
  }
989
1135
  };
990
1136
  return workersInstance;
@@ -998,44 +1144,17 @@ var WorkerService = class {
998
1144
  })
999
1145
  );
1000
1146
  }
1001
- async startANewWorker(config, workersInstance, workerIndex = workersInstance.workers.length) {
1002
- if (!workersInstance.running) {
1003
- return;
1004
- }
1005
- const workerInstance = {
1006
- running: true,
1007
- workerIndex,
1008
- stop: async () => {
1009
- import_logger4.logger.info(`Stopping worker [w${workerIndex}]...`);
1010
- workerInstance.running = false;
1011
- await workerInstance.promise;
1012
- },
1013
- respawn: async () => {
1014
- import_logger4.logger.info(`Respawning worker [w${workerIndex}]...`);
1015
- workerInstance.running = false;
1016
- await this.startANewWorker(config, workersInstance, workerIndex);
1017
- }
1018
- };
1019
- const workerPromise = this.startWorker(config, workersInstance, workerInstance);
1020
- workerInstance.promise = workerPromise;
1021
- workersInstance.workers[workerIndex] = workerInstance;
1022
- }
1023
- async runWorkers(config, workersInstance) {
1147
+ async runScheduler(config, workersInstance) {
1024
1148
  import_logger4.logger.debug("Will ensure maxTriesReached retention and TTL index");
1025
- const ensureRetentionPromise = this.jobsRepo.ensureMaxTriesReachedRetention(config.maxTriesReachedRetentionMs).catch((error) => import_logger4.logger.error("Error ensuring maxTriesReached retention", { error }));
1149
+ void this.jobsRepo.ensureMaxTriesReachedRetention(config.maxTriesReachedRetentionMs).catch((error) => import_logger4.logger.error("Error ensuring maxTriesReached retention", { error }));
1026
1150
  import_logger4.logger.debug("Will ensure records for recurrent jobs");
1027
1151
  await this.ensureRecords(config);
1028
- const workersCount = config.workersCount;
1029
- const workerWord = workersCount === 1 ? "worker" : "workers";
1030
- import_logger4.logger.info(`Starting ${workersCount} ${workerWord}`);
1031
- for (let workerIndex = 0; workerIndex < workersCount; workerIndex++) {
1032
- this.startANewWorker(config, workersInstance, workerIndex);
1033
- }
1034
- await ensureRetentionPromise;
1152
+ import_logger4.logger.info(`Starting job scheduler with concurrency ${config.workersCount}`);
1153
+ await this.runSchedulerLoop(config, workersInstance);
1035
1154
  }
1036
1155
  /**
1037
- * Starts the job workers with the provided configuration.
1038
- * @param userConfig - Configuration for the workers. Required field: jobs
1156
+ * Starts one job scheduler with the provided concurrency configuration.
1157
+ * @param userConfig - Configuration for the scheduler. Required field: jobs
1039
1158
  */
1040
1159
  startWorkers(userConfig) {
1041
1160
  const maxTriesReachedRetentionMs = userConfig.maxTriesReachedRetentionMs === void 0 ? DEFAULT_MAX_TRIES_REACHED_RETENTION_MS : userConfig.maxTriesReachedRetentionMs;
@@ -1043,7 +1162,6 @@ var WorkerService = class {
1043
1162
  throw new Error("maxTriesReachedRetentionMs must be null, zero, or a positive number");
1044
1163
  }
1045
1164
  const config = {
1046
- cooldownPeriod: 100,
1047
1165
  pollInterval: 3e3,
1048
1166
  workersCount: 4,
1049
1167
  defaultLockTime: 30 * 1e3,
@@ -1052,8 +1170,11 @@ var WorkerService = class {
1052
1170
  };
1053
1171
  setNameToJobs(config.jobs);
1054
1172
  const workersInstance = this.createWorkersInstanceDefinition(config);
1055
- import_logger4.logger.debug("Starting workers", config);
1056
- this.runWorkers(config, workersInstance);
1173
+ import_logger4.logger.debug("Starting job scheduler", config);
1174
+ workersInstance.schedulerPromise = this.runScheduler(config, workersInstance).catch((error) => {
1175
+ workersInstance.running = false;
1176
+ import_logger4.logger.error("Job scheduler stopped after an unexpected error.", { error });
1177
+ });
1057
1178
  return workersInstance;
1058
1179
  }
1059
1180
  };