@orion-js/dogs 4.5.1 → 4.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +277 -156
- package/dist/index.cjs.map +1 -1
- package/dist/index.js +277 -156
- package/dist/index.js.map +1 -1
- package/dist/repos/JobsRepo.d.ts +7 -6
- package/dist/services/Executor.d.ts +5 -5
- package/dist/services/WorkerService.d.ts +30 -13
- package/dist/types/JobRecord.d.ts +8 -0
- package/dist/types/StartConfig.d.ts +4 -3
- package/dist/types/Worker.d.ts +6 -11
- package/package.json +1 -1
package/dist/index.cjs
CHANGED
|
@@ -115,6 +115,11 @@ var JobRecordSchema = (0, import_schema.schemaWithName)("JobRecord", {
|
|
|
115
115
|
nextRunAt: { type: "date" },
|
|
116
116
|
lastRunAt: { type: "date", optional: true },
|
|
117
117
|
lockedUntil: { type: "date", optional: true },
|
|
118
|
+
/**
|
|
119
|
+
* Identifies the execution that currently owns the lock. Mutations from an execution must match
|
|
120
|
+
* this value so a stale execution cannot modify a newer claim.
|
|
121
|
+
*/
|
|
122
|
+
lockId: { type: "string", optional: true },
|
|
118
123
|
tries: { type: "number", optional: true },
|
|
119
124
|
params: { type: "blackbox", optional: true },
|
|
120
125
|
/**
|
|
@@ -182,6 +187,7 @@ var JobsRepo = class {
|
|
|
182
187
|
__publicField(this, "jobs", __runInitializers(_init, 8, this)), __runInitializers(_init, 11, this);
|
|
183
188
|
}
|
|
184
189
|
async getJobAndLock(jobNames, lockTime) {
|
|
190
|
+
const executionId = (0, import_helpers.generateId)();
|
|
185
191
|
const lockedUntil = new Date(Date.now() + lockTime);
|
|
186
192
|
const job = await this.jobs.findOneAndUpdate(
|
|
187
193
|
{
|
|
@@ -195,7 +201,7 @@ var JobsRepo = class {
|
|
|
195
201
|
]
|
|
196
202
|
},
|
|
197
203
|
{
|
|
198
|
-
$set: { lockedUntil, lastRunAt: /* @__PURE__ */ new Date() },
|
|
204
|
+
$set: { lockId: executionId, lockedUntil, lastRunAt: /* @__PURE__ */ new Date() },
|
|
199
205
|
$inc: { tries: 1 }
|
|
200
206
|
},
|
|
201
207
|
{
|
|
@@ -216,7 +222,8 @@ var JobsRepo = class {
|
|
|
216
222
|
}
|
|
217
223
|
return {
|
|
218
224
|
jobId: job._id,
|
|
219
|
-
executionId
|
|
225
|
+
executionId,
|
|
226
|
+
lockId: executionId,
|
|
220
227
|
name: job.jobName,
|
|
221
228
|
params: job.params,
|
|
222
229
|
type: job.type,
|
|
@@ -227,8 +234,11 @@ var JobsRepo = class {
|
|
|
227
234
|
wasStale
|
|
228
235
|
};
|
|
229
236
|
}
|
|
230
|
-
async setJobRecordPriority(jobId, priority) {
|
|
231
|
-
await this.jobs.updateOne(
|
|
237
|
+
async setJobRecordPriority(jobId, priority, lockId) {
|
|
238
|
+
const result = await this.jobs.updateOne(lockId ? { _id: jobId, lockId } : jobId, {
|
|
239
|
+
$set: { priority }
|
|
240
|
+
});
|
|
241
|
+
return result.matchedCount > 0;
|
|
232
242
|
}
|
|
233
243
|
async scheduleNextRun(options) {
|
|
234
244
|
const updator = {
|
|
@@ -237,22 +247,31 @@ var JobsRepo = class {
|
|
|
237
247
|
priority: options.priority,
|
|
238
248
|
...options.resetTries ? { tries: 0 } : {}
|
|
239
249
|
},
|
|
240
|
-
$unset: { lockedUntil: "" }
|
|
250
|
+
$unset: { lockId: "", lockedUntil: "" }
|
|
241
251
|
};
|
|
242
|
-
await this.jobs.updateOne(
|
|
252
|
+
const result = await this.jobs.updateOne(
|
|
253
|
+
options.lockId ? { _id: options.jobId, lockId: options.lockId } : options.jobId,
|
|
254
|
+
updator
|
|
255
|
+
);
|
|
256
|
+
return result.matchedCount > 0;
|
|
243
257
|
}
|
|
244
|
-
async deleteEventJob(jobId) {
|
|
245
|
-
await this.jobs.deleteOne({
|
|
258
|
+
async deleteEventJob(jobId, lockId) {
|
|
259
|
+
const result = await this.jobs.deleteOne({
|
|
260
|
+
_id: jobId,
|
|
261
|
+
type: "event",
|
|
262
|
+
...lockId ? { lockId } : {}
|
|
263
|
+
});
|
|
264
|
+
return result.deletedCount > 0;
|
|
246
265
|
}
|
|
247
266
|
/**
|
|
248
267
|
* Marks an event job as having reached its maximum tries limit.
|
|
249
268
|
* The job won't be picked up for execution and MongoDB will delete it after
|
|
250
269
|
* the configured retention period.
|
|
251
270
|
*/
|
|
252
|
-
async markJobAsMaxTriesReached(jobId, retentionMs = DEFAULT_MAX_TRIES_REACHED_RETENTION_MS) {
|
|
271
|
+
async markJobAsMaxTriesReached(jobId, retentionMs = DEFAULT_MAX_TRIES_REACHED_RETENTION_MS, lockId) {
|
|
253
272
|
const maxTriesReachedAt = /* @__PURE__ */ new Date();
|
|
254
|
-
await this.jobs.updateOne(
|
|
255
|
-
{ _id: jobId, type: "event" },
|
|
273
|
+
const result = await this.jobs.updateOne(
|
|
274
|
+
{ _id: jobId, type: "event", ...lockId ? { lockId } : {} },
|
|
256
275
|
{
|
|
257
276
|
$set: {
|
|
258
277
|
status: "maxTriesReached",
|
|
@@ -260,11 +279,13 @@ var JobsRepo = class {
|
|
|
260
279
|
...retentionMs === null ? {} : { expiresAt: new Date(maxTriesReachedAt.getTime() + retentionMs) }
|
|
261
280
|
},
|
|
262
281
|
$unset: {
|
|
282
|
+
lockId: "",
|
|
263
283
|
lockedUntil: "",
|
|
264
284
|
...retentionMs === null ? { expiresAt: "" } : {}
|
|
265
285
|
}
|
|
266
286
|
}
|
|
267
287
|
);
|
|
288
|
+
return result.matchedCount > 0;
|
|
268
289
|
}
|
|
269
290
|
/**
|
|
270
291
|
* Applies the current retention configuration to existing maxTriesReached
|
|
@@ -293,23 +314,22 @@ var JobsRepo = class {
|
|
|
293
314
|
}
|
|
294
315
|
]);
|
|
295
316
|
}
|
|
296
|
-
async extendLockTime(jobId, extraTime) {
|
|
297
|
-
|
|
317
|
+
async extendLockTime(jobId, extraTime, lockId) {
|
|
318
|
+
return this.updateLockTime(jobId, extraTime, lockId);
|
|
298
319
|
}
|
|
299
320
|
/**
|
|
300
321
|
* Updates the lock time for a job to the specified duration from now.
|
|
301
322
|
* Can be used to both extend or shorten the lock time.
|
|
302
323
|
*/
|
|
303
|
-
async updateLockTime(jobId, lockDuration) {
|
|
324
|
+
async updateLockTime(jobId, lockDuration, lockId) {
|
|
304
325
|
const lockedUntil = new Date(Date.now() + lockDuration);
|
|
305
|
-
await this.jobs.updateOne(
|
|
306
|
-
{
|
|
307
|
-
_id: jobId
|
|
308
|
-
},
|
|
326
|
+
const result = await this.jobs.updateOne(
|
|
327
|
+
{ _id: jobId, ...lockId ? { lockId } : {} },
|
|
309
328
|
{
|
|
310
329
|
$set: { lockedUntil }
|
|
311
330
|
}
|
|
312
331
|
);
|
|
332
|
+
return result.matchedCount > 0;
|
|
313
333
|
}
|
|
314
334
|
async unlockAllJobs() {
|
|
315
335
|
const result = await this.jobs.updateMany(
|
|
@@ -317,7 +337,7 @@ var JobsRepo = class {
|
|
|
317
337
|
lockedUntil: { $exists: true }
|
|
318
338
|
},
|
|
319
339
|
{
|
|
320
|
-
$unset: { lockedUntil: "" }
|
|
340
|
+
$unset: { lockId: "", lockedUntil: "" }
|
|
321
341
|
}
|
|
322
342
|
);
|
|
323
343
|
return result.modifiedCount;
|
|
@@ -623,11 +643,15 @@ var Executor = class {
|
|
|
623
643
|
getEffectiveLockTime(job, jobToRun) {
|
|
624
644
|
return job.lockTime ?? jobToRun.lockTime;
|
|
625
645
|
}
|
|
626
|
-
getContext(job, jobToRun, onStale) {
|
|
646
|
+
getContext(job, jobToRun, onStale, isExecutionActive) {
|
|
627
647
|
var _a;
|
|
628
648
|
const effectiveLockTime = this.getEffectiveLockTime(job, jobToRun);
|
|
629
649
|
let staleTimeout = setTimeout(() => onStale(), effectiveLockTime);
|
|
630
650
|
(_a = staleTimeout.unref) == null ? void 0 : _a.call(staleTimeout);
|
|
651
|
+
const contextLogger = import_logger3.logger.addMetadata({
|
|
652
|
+
jobName: jobToRun.name,
|
|
653
|
+
jobId: jobToRun.jobId
|
|
654
|
+
});
|
|
631
655
|
return {
|
|
632
656
|
definition: job,
|
|
633
657
|
record: jobToRun,
|
|
@@ -635,15 +659,26 @@ var Executor = class {
|
|
|
635
659
|
clearStaleTimeout: () => clearTimeout(staleTimeout),
|
|
636
660
|
extendLockTime: async (extraTime) => {
|
|
637
661
|
var _a2;
|
|
662
|
+
if (!isExecutionActive()) {
|
|
663
|
+
contextLogger.warn(`Will not extend lock for stale execution "${jobToRun.executionId}"`);
|
|
664
|
+
return;
|
|
665
|
+
}
|
|
638
666
|
clearTimeout(staleTimeout);
|
|
667
|
+
let didExtend = false;
|
|
668
|
+
try {
|
|
669
|
+
didExtend = await this.jobsRepo.extendLockTime(jobToRun.jobId, extraTime, jobToRun.lockId);
|
|
670
|
+
} catch (error) {
|
|
671
|
+
onStale();
|
|
672
|
+
throw error;
|
|
673
|
+
}
|
|
674
|
+
if (!didExtend) {
|
|
675
|
+
onStale();
|
|
676
|
+
return;
|
|
677
|
+
}
|
|
639
678
|
staleTimeout = setTimeout(() => onStale(), extraTime);
|
|
640
679
|
(_a2 = staleTimeout.unref) == null ? void 0 : _a2.call(staleTimeout);
|
|
641
|
-
await this.jobsRepo.extendLockTime(jobToRun.jobId, extraTime);
|
|
642
680
|
},
|
|
643
|
-
logger:
|
|
644
|
-
jobName: jobToRun.name,
|
|
645
|
-
jobId: jobToRun.jobId
|
|
646
|
-
})
|
|
681
|
+
logger: contextLogger
|
|
647
682
|
};
|
|
648
683
|
}
|
|
649
684
|
getJobDefinition(jobToRun, jobs) {
|
|
@@ -675,8 +710,16 @@ var Executor = class {
|
|
|
675
710
|
jobLogger.warn(
|
|
676
711
|
`Job "${jobToRun.name}" has exceeded max tries (${jobToRun.tries}). Marking as maxTriesReached.`
|
|
677
712
|
);
|
|
678
|
-
await this.jobsRepo.markJobAsMaxTriesReached(
|
|
679
|
-
|
|
713
|
+
const didMark = await this.jobsRepo.markJobAsMaxTriesReached(
|
|
714
|
+
jobToRun.jobId,
|
|
715
|
+
retentionMs,
|
|
716
|
+
jobToRun.lockId
|
|
717
|
+
);
|
|
718
|
+
if (!didMark) {
|
|
719
|
+
jobLogger.warn(`Could not mark job "${jobToRun.name}" because its lock is no longer owned`);
|
|
720
|
+
return false;
|
|
721
|
+
}
|
|
722
|
+
if (!onMaxTriesReached) return true;
|
|
680
723
|
try {
|
|
681
724
|
await onMaxTriesReached(jobToRun);
|
|
682
725
|
} catch (callbackError) {
|
|
@@ -684,30 +727,33 @@ var Executor = class {
|
|
|
684
727
|
error: callbackError
|
|
685
728
|
});
|
|
686
729
|
}
|
|
730
|
+
return true;
|
|
687
731
|
}
|
|
688
732
|
async onError(error, job, jobToRun, context) {
|
|
689
733
|
const scheduleRecurrent = async () => {
|
|
690
734
|
if (job.type === "recurrent") {
|
|
691
|
-
|
|
735
|
+
return this.jobsRepo.scheduleNextRun({
|
|
692
736
|
jobId: jobToRun.jobId,
|
|
693
737
|
nextRunAt: getNextRunDate(job),
|
|
694
738
|
resetTries: true,
|
|
695
|
-
priority: job.priority
|
|
739
|
+
priority: job.priority,
|
|
740
|
+
lockId: jobToRun.lockId
|
|
696
741
|
});
|
|
697
742
|
}
|
|
743
|
+
return true;
|
|
698
744
|
};
|
|
699
745
|
const handleRetry = async (nextRunAt) => {
|
|
700
|
-
|
|
746
|
+
return this.jobsRepo.scheduleNextRun({
|
|
701
747
|
jobId: jobToRun.jobId,
|
|
702
748
|
nextRunAt,
|
|
703
749
|
resetTries: false,
|
|
704
|
-
priority: job.type === "recurrent" ? job.priority : jobToRun.priority
|
|
750
|
+
priority: job.type === "recurrent" ? job.priority : jobToRun.priority,
|
|
751
|
+
lockId: jobToRun.lockId
|
|
705
752
|
});
|
|
706
753
|
};
|
|
707
754
|
if (!job.onError) {
|
|
708
755
|
context.logger.error(`Error executing job "${jobToRun.name}"`, { error });
|
|
709
|
-
|
|
710
|
-
return;
|
|
756
|
+
return scheduleRecurrent();
|
|
711
757
|
}
|
|
712
758
|
context.logger.info(`Error executing job "${jobToRun.name}"`, { error });
|
|
713
759
|
const result = await job.onError(
|
|
@@ -716,12 +762,12 @@ var Executor = class {
|
|
|
716
762
|
context
|
|
717
763
|
);
|
|
718
764
|
if (result.action === "dismiss") {
|
|
719
|
-
|
|
720
|
-
return;
|
|
765
|
+
return scheduleRecurrent();
|
|
721
766
|
}
|
|
722
767
|
if (result.action === "retry") {
|
|
723
|
-
|
|
768
|
+
return handleRetry(getNextRunDate(result));
|
|
724
769
|
}
|
|
770
|
+
return true;
|
|
725
771
|
}
|
|
726
772
|
async saveExecution(options) {
|
|
727
773
|
const { startedAt, status, errorMessage, result, job, jobToRun } = options;
|
|
@@ -751,19 +797,21 @@ var Executor = class {
|
|
|
751
797
|
async afterExecutionSuccess(job, jobToRun, context) {
|
|
752
798
|
if (job.type === "recurrent") {
|
|
753
799
|
context.logger.debug(`Scheduling next run for recurrent job "${jobToRun.name}"`);
|
|
754
|
-
|
|
800
|
+
return this.jobsRepo.scheduleNextRun({
|
|
755
801
|
jobId: jobToRun.jobId,
|
|
756
802
|
nextRunAt: getNextRunDate(job),
|
|
757
803
|
resetTries: true,
|
|
758
|
-
priority: job.priority
|
|
804
|
+
priority: job.priority,
|
|
805
|
+
lockId: jobToRun.lockId
|
|
759
806
|
});
|
|
760
807
|
}
|
|
761
808
|
if (job.type === "event") {
|
|
762
809
|
context.logger.debug(`Removing event job after success "${jobToRun.name}"`);
|
|
763
|
-
|
|
810
|
+
return this.jobsRepo.deleteEventJob(jobToRun.jobId, jobToRun.lockId);
|
|
764
811
|
}
|
|
812
|
+
return false;
|
|
765
813
|
}
|
|
766
|
-
async executeJob(config, jobToRun,
|
|
814
|
+
async executeJob(config, jobToRun, onExecutionStale) {
|
|
767
815
|
const job = this.getJobDefinition(jobToRun, config.jobs);
|
|
768
816
|
if (!job) return;
|
|
769
817
|
if (job.type === "event") {
|
|
@@ -779,33 +827,64 @@ var Executor = class {
|
|
|
779
827
|
}
|
|
780
828
|
const effectiveLockTime = this.getEffectiveLockTime(job, jobToRun);
|
|
781
829
|
if (effectiveLockTime !== jobToRun.lockTime) {
|
|
782
|
-
await this.jobsRepo.updateLockTime(
|
|
830
|
+
const didUpdateLock = await this.jobsRepo.updateLockTime(
|
|
831
|
+
jobToRun.jobId,
|
|
832
|
+
effectiveLockTime,
|
|
833
|
+
jobToRun.lockId
|
|
834
|
+
);
|
|
835
|
+
if (!didUpdateLock) {
|
|
836
|
+
import_logger3.logger.warn(`Will not execute job "${jobToRun.name}" because its lock is no longer owned`);
|
|
837
|
+
return;
|
|
838
|
+
}
|
|
783
839
|
}
|
|
784
840
|
const tracer = import_api.trace.getTracer("orionjs.dogs", "1.0");
|
|
785
841
|
await tracer.startActiveSpan(`job.${jobToRun.name}.${jobToRun.executionId}`, async (span) => {
|
|
786
842
|
try {
|
|
787
843
|
const startedAt = /* @__PURE__ */ new Date();
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
844
|
+
let executionStatus = "running";
|
|
845
|
+
let staleHandlingPromise;
|
|
846
|
+
let context;
|
|
847
|
+
const markAsStale = () => {
|
|
848
|
+
if (executionStatus === "stale") return staleHandlingPromise;
|
|
849
|
+
executionStatus = "stale";
|
|
850
|
+
context.clearStaleTimeout();
|
|
851
|
+
staleHandlingPromise = (async () => {
|
|
852
|
+
context.logger.warn(`Job "${jobToRun.name}" is stale`);
|
|
853
|
+
if (job.onStale) {
|
|
854
|
+
void Promise.resolve(job.onStale(jobToRun.params, context)).catch((error) => {
|
|
855
|
+
context.logger.error(`Error in onStale callback for job "${jobToRun.name}"`, {
|
|
856
|
+
error
|
|
857
|
+
});
|
|
858
|
+
});
|
|
859
|
+
}
|
|
860
|
+
try {
|
|
861
|
+
await this.jobsRepo.setJobRecordPriority(jobToRun.jobId, 0, jobToRun.lockId);
|
|
862
|
+
} catch (error) {
|
|
863
|
+
context.logger.error(`Error lowering priority for stale job "${jobToRun.name}"`, {
|
|
864
|
+
error
|
|
865
|
+
});
|
|
866
|
+
} finally {
|
|
867
|
+
onExecutionStale();
|
|
868
|
+
}
|
|
869
|
+
await this.saveExecution({
|
|
870
|
+
startedAt,
|
|
871
|
+
status: "stale",
|
|
872
|
+
result: null,
|
|
873
|
+
errorMessage: null,
|
|
874
|
+
job,
|
|
875
|
+
jobToRun
|
|
876
|
+
});
|
|
877
|
+
})().catch((error) => {
|
|
878
|
+
context.logger.error("Error handling stale execution", { error });
|
|
806
879
|
});
|
|
880
|
+
return staleHandlingPromise;
|
|
807
881
|
};
|
|
808
|
-
|
|
882
|
+
context = this.getContext(
|
|
883
|
+
job,
|
|
884
|
+
jobToRun,
|
|
885
|
+
() => void markAsStale(),
|
|
886
|
+
() => executionStatus === "running"
|
|
887
|
+
);
|
|
809
888
|
const extraContext = {
|
|
810
889
|
controllerType: "job",
|
|
811
890
|
jobName: jobToRun.name,
|
|
@@ -821,6 +900,18 @@ var Executor = class {
|
|
|
821
900
|
});
|
|
822
901
|
const result = await job.resolve(jobToRun.params, context);
|
|
823
902
|
context.clearStaleTimeout();
|
|
903
|
+
if (executionStatus === "stale") {
|
|
904
|
+
await staleHandlingPromise;
|
|
905
|
+
context.logger.warn(
|
|
906
|
+
`Ignoring late success from stale execution "${jobToRun.executionId}"`
|
|
907
|
+
);
|
|
908
|
+
return;
|
|
909
|
+
}
|
|
910
|
+
const stillOwnsJob = await this.afterExecutionSuccess(job, jobToRun, context);
|
|
911
|
+
if (!stillOwnsJob) {
|
|
912
|
+
await markAsStale();
|
|
913
|
+
return;
|
|
914
|
+
}
|
|
824
915
|
void this.saveExecution({
|
|
825
916
|
startedAt,
|
|
826
917
|
status: "success",
|
|
@@ -831,9 +922,21 @@ var Executor = class {
|
|
|
831
922
|
}).catch((error) => {
|
|
832
923
|
context.logger.error("Error saving successful execution history", { error });
|
|
833
924
|
});
|
|
834
|
-
await this.afterExecutionSuccess(job, jobToRun, context);
|
|
835
925
|
} catch (error) {
|
|
836
926
|
context.clearStaleTimeout();
|
|
927
|
+
if (executionStatus === "stale") {
|
|
928
|
+
await staleHandlingPromise;
|
|
929
|
+
context.logger.warn(
|
|
930
|
+
`Ignoring late error from stale execution "${jobToRun.executionId}"`,
|
|
931
|
+
{ error }
|
|
932
|
+
);
|
|
933
|
+
return;
|
|
934
|
+
}
|
|
935
|
+
const stillOwnsJob = await this.onError(error, job, jobToRun, context);
|
|
936
|
+
if (!stillOwnsJob) {
|
|
937
|
+
await markAsStale();
|
|
938
|
+
return;
|
|
939
|
+
}
|
|
837
940
|
void this.saveExecution({
|
|
838
941
|
startedAt,
|
|
839
942
|
status: "error",
|
|
@@ -844,7 +947,6 @@ var Executor = class {
|
|
|
844
947
|
}).catch((saveError) => {
|
|
845
948
|
context.logger.error("Error saving failed execution history", { error: saveError });
|
|
846
949
|
});
|
|
847
|
-
await this.onError(error, job, jobToRun, context);
|
|
848
950
|
}
|
|
849
951
|
});
|
|
850
952
|
} catch (error) {
|
|
@@ -904,87 +1006,131 @@ var WorkerService = class {
|
|
|
904
1006
|
}
|
|
905
1007
|
workersInstance.runningJobsByName.set(jobName, currentExecutions - 1);
|
|
906
1008
|
}
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
workersInstance.
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
return await callback();
|
|
916
|
-
} finally {
|
|
917
|
-
releaseLock();
|
|
1009
|
+
releaseExecutionCapacity(workersInstance, execution, nextStatus) {
|
|
1010
|
+
if (execution.status !== "running") return;
|
|
1011
|
+
execution.status = nextStatus;
|
|
1012
|
+
workersInstance.activeExecutions.delete(execution.job.executionId);
|
|
1013
|
+
this.releaseJobExecution(workersInstance, execution.job.name);
|
|
1014
|
+
execution.releaseCapacity();
|
|
1015
|
+
if (nextStatus === "stale") {
|
|
1016
|
+
workersInstance.staleExecutions.set(execution.job.executionId, execution);
|
|
918
1017
|
}
|
|
919
1018
|
}
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
1019
|
+
startExecution(config, workersInstance, jobToRun) {
|
|
1020
|
+
let releaseCapacity;
|
|
1021
|
+
const capacityPromise = new Promise((resolve) => {
|
|
1022
|
+
releaseCapacity = resolve;
|
|
1023
|
+
});
|
|
1024
|
+
const execution = {
|
|
1025
|
+
job: jobToRun,
|
|
1026
|
+
promise: Promise.resolve(),
|
|
1027
|
+
capacityPromise,
|
|
1028
|
+
releaseCapacity,
|
|
1029
|
+
status: "running"
|
|
1030
|
+
};
|
|
1031
|
+
this.reserveJobExecution(workersInstance, jobToRun.name);
|
|
1032
|
+
workersInstance.activeExecutions.set(jobToRun.executionId, execution);
|
|
1033
|
+
import_logger4.logger.debug(`Starting job execution "${jobToRun.executionId}"`, jobToRun);
|
|
1034
|
+
const executionPromise = this.executor.executeJob(config, jobToRun, () => {
|
|
1035
|
+
this.releaseExecutionCapacity(workersInstance, execution, "stale");
|
|
1036
|
+
});
|
|
1037
|
+
execution.promise = executionPromise;
|
|
1038
|
+
void executionPromise.catch((error) => {
|
|
1039
|
+
import_logger4.logger.error(`Unhandled error executing job "${jobToRun.name}"`, { error });
|
|
1040
|
+
}).finally(() => {
|
|
1041
|
+
if (execution.status === "running") {
|
|
1042
|
+
this.releaseExecutionCapacity(workersInstance, execution, "finished");
|
|
1043
|
+
} else if (execution.status === "stale") {
|
|
1044
|
+
workersInstance.staleExecutions.delete(jobToRun.executionId);
|
|
1045
|
+
}
|
|
928
1046
|
});
|
|
929
1047
|
}
|
|
930
|
-
async
|
|
931
|
-
|
|
932
|
-
if (!jobToRun) {
|
|
933
|
-
import_logger4.logger.debug("No job to run");
|
|
934
|
-
return false;
|
|
935
|
-
}
|
|
936
|
-
import_logger4.logger.debug(`Got job [w${workerInstance.workerIndex}] to run:`, jobToRun);
|
|
937
|
-
try {
|
|
938
|
-
await this.executor.executeJob(executeConfig, jobToRun, workerInstance.respawn);
|
|
939
|
-
} finally {
|
|
940
|
-
this.releaseJobExecution(workersInstance, jobToRun.name);
|
|
941
|
-
}
|
|
942
|
-
return true;
|
|
1048
|
+
async waitForPollOrStop(config, workersInstance) {
|
|
1049
|
+
await Promise.race([(0, import_helpers2.sleep)(config.pollInterval), workersInstance.stopSignal]);
|
|
943
1050
|
}
|
|
944
|
-
async
|
|
945
|
-
const
|
|
946
|
-
|
|
947
|
-
|
|
1051
|
+
async waitForCapacityOrStop(workersInstance) {
|
|
1052
|
+
const capacityPromises = Array.from(
|
|
1053
|
+
workersInstance.activeExecutions.values(),
|
|
1054
|
+
(execution) => execution.capacityPromise
|
|
948
1055
|
);
|
|
1056
|
+
if (capacityPromises.length === 0) return;
|
|
1057
|
+
await Promise.race([...capacityPromises, workersInstance.stopSignal]);
|
|
1058
|
+
}
|
|
1059
|
+
async runSchedulerLoop(config, workersInstance) {
|
|
1060
|
+
const jobNames = this.getJobNames(config.jobs);
|
|
949
1061
|
const executeConfig = {
|
|
950
1062
|
jobs: config.jobs,
|
|
951
1063
|
maxTries: config.maxTries,
|
|
952
1064
|
maxTriesReachedRetentionMs: config.maxTriesReachedRetentionMs,
|
|
953
1065
|
onMaxTriesReached: config.onMaxTriesReached
|
|
954
1066
|
};
|
|
955
|
-
|
|
956
|
-
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
}
|
|
1067
|
+
import_logger4.logger.debug(`Running scheduler loop for jobs "${jobNames.join(", ")}"...`);
|
|
1068
|
+
while (workersInstance.running) {
|
|
1069
|
+
let didMiss = false;
|
|
1070
|
+
let allJobNamesAtCapacity = false;
|
|
960
1071
|
try {
|
|
961
|
-
|
|
962
|
-
config,
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
1072
|
+
while (workersInstance.activeExecutions.size < config.workersCount) {
|
|
1073
|
+
const availableJobNames = this.getAvailableJobNames(config, workersInstance, jobNames);
|
|
1074
|
+
if (availableJobNames.length === 0) {
|
|
1075
|
+
allJobNamesAtCapacity = workersInstance.activeExecutions.size > 0;
|
|
1076
|
+
break;
|
|
1077
|
+
}
|
|
1078
|
+
const jobToRun = await this.jobsRepo.getJobAndLock(
|
|
1079
|
+
availableJobNames,
|
|
1080
|
+
config.defaultLockTime
|
|
1081
|
+
);
|
|
1082
|
+
if (!jobToRun) {
|
|
1083
|
+
import_logger4.logger.debug("No job to run");
|
|
1084
|
+
didMiss = true;
|
|
1085
|
+
break;
|
|
1086
|
+
}
|
|
1087
|
+
this.startExecution(executeConfig, workersInstance, jobToRun);
|
|
1088
|
+
if (!workersInstance.running) break;
|
|
1089
|
+
}
|
|
970
1090
|
} catch (error) {
|
|
971
|
-
import_logger4.logger.error("Error in job
|
|
972
|
-
|
|
1091
|
+
import_logger4.logger.error("Error in job scheduler.", { error });
|
|
1092
|
+
didMiss = true;
|
|
1093
|
+
}
|
|
1094
|
+
if (!workersInstance.running) return;
|
|
1095
|
+
const concurrencyIsFull = workersInstance.activeExecutions.size >= config.workersCount && workersInstance.activeExecutions.size > 0;
|
|
1096
|
+
if (concurrencyIsFull || allJobNamesAtCapacity) {
|
|
1097
|
+
await this.waitForCapacityOrStop(workersInstance);
|
|
1098
|
+
} else if (didMiss || workersInstance.activeExecutions.size === 0) {
|
|
1099
|
+
await this.waitForPollOrStop(config, workersInstance);
|
|
973
1100
|
}
|
|
974
1101
|
}
|
|
975
1102
|
}
|
|
976
1103
|
createWorkersInstanceDefinition(config) {
|
|
1104
|
+
let signalStop;
|
|
1105
|
+
const stopSignal = new Promise((resolve) => {
|
|
1106
|
+
signalStop = resolve;
|
|
1107
|
+
});
|
|
977
1108
|
const workersInstance = {
|
|
978
1109
|
running: true,
|
|
979
1110
|
workersCount: config.workersCount,
|
|
980
|
-
|
|
1111
|
+
activeExecutions: /* @__PURE__ */ new Map(),
|
|
1112
|
+
staleExecutions: /* @__PURE__ */ new Map(),
|
|
981
1113
|
runningJobsByName: /* @__PURE__ */ new Map(),
|
|
982
|
-
|
|
1114
|
+
schedulerPromise: Promise.resolve(),
|
|
1115
|
+
stopSignal,
|
|
1116
|
+
signalStop,
|
|
1117
|
+
get runningExecutions() {
|
|
1118
|
+
return this.activeExecutions.size;
|
|
1119
|
+
},
|
|
983
1120
|
stop: async () => {
|
|
984
|
-
|
|
985
|
-
workersInstance.
|
|
986
|
-
|
|
987
|
-
|
|
1121
|
+
if (workersInstance.stopPromise) return workersInstance.stopPromise;
|
|
1122
|
+
workersInstance.stopPromise = (async () => {
|
|
1123
|
+
import_logger4.logger.info("Stopping job scheduler...");
|
|
1124
|
+
workersInstance.running = false;
|
|
1125
|
+
workersInstance.signalStop();
|
|
1126
|
+
await workersInstance.schedulerPromise;
|
|
1127
|
+
const activeCapacity = Array.from(
|
|
1128
|
+
workersInstance.activeExecutions.values(),
|
|
1129
|
+
(execution) => execution.capacityPromise
|
|
1130
|
+
);
|
|
1131
|
+
await Promise.all(activeCapacity);
|
|
1132
|
+
})();
|
|
1133
|
+
return workersInstance.stopPromise;
|
|
988
1134
|
}
|
|
989
1135
|
};
|
|
990
1136
|
return workersInstance;
|
|
@@ -998,44 +1144,17 @@ var WorkerService = class {
|
|
|
998
1144
|
})
|
|
999
1145
|
);
|
|
1000
1146
|
}
|
|
1001
|
-
async
|
|
1002
|
-
if (!workersInstance.running) {
|
|
1003
|
-
return;
|
|
1004
|
-
}
|
|
1005
|
-
const workerInstance = {
|
|
1006
|
-
running: true,
|
|
1007
|
-
workerIndex,
|
|
1008
|
-
stop: async () => {
|
|
1009
|
-
import_logger4.logger.info(`Stopping worker [w${workerIndex}]...`);
|
|
1010
|
-
workerInstance.running = false;
|
|
1011
|
-
await workerInstance.promise;
|
|
1012
|
-
},
|
|
1013
|
-
respawn: async () => {
|
|
1014
|
-
import_logger4.logger.info(`Respawning worker [w${workerIndex}]...`);
|
|
1015
|
-
workerInstance.running = false;
|
|
1016
|
-
await this.startANewWorker(config, workersInstance, workerIndex);
|
|
1017
|
-
}
|
|
1018
|
-
};
|
|
1019
|
-
const workerPromise = this.startWorker(config, workersInstance, workerInstance);
|
|
1020
|
-
workerInstance.promise = workerPromise;
|
|
1021
|
-
workersInstance.workers[workerIndex] = workerInstance;
|
|
1022
|
-
}
|
|
1023
|
-
async runWorkers(config, workersInstance) {
|
|
1147
|
+
async runScheduler(config, workersInstance) {
|
|
1024
1148
|
import_logger4.logger.debug("Will ensure maxTriesReached retention and TTL index");
|
|
1025
|
-
|
|
1149
|
+
void this.jobsRepo.ensureMaxTriesReachedRetention(config.maxTriesReachedRetentionMs).catch((error) => import_logger4.logger.error("Error ensuring maxTriesReached retention", { error }));
|
|
1026
1150
|
import_logger4.logger.debug("Will ensure records for recurrent jobs");
|
|
1027
1151
|
await this.ensureRecords(config);
|
|
1028
|
-
|
|
1029
|
-
|
|
1030
|
-
import_logger4.logger.info(`Starting ${workersCount} ${workerWord}`);
|
|
1031
|
-
for (let workerIndex = 0; workerIndex < workersCount; workerIndex++) {
|
|
1032
|
-
this.startANewWorker(config, workersInstance, workerIndex);
|
|
1033
|
-
}
|
|
1034
|
-
await ensureRetentionPromise;
|
|
1152
|
+
import_logger4.logger.info(`Starting job scheduler with concurrency ${config.workersCount}`);
|
|
1153
|
+
await this.runSchedulerLoop(config, workersInstance);
|
|
1035
1154
|
}
|
|
1036
1155
|
/**
|
|
1037
|
-
* Starts
|
|
1038
|
-
* @param userConfig - Configuration for the
|
|
1156
|
+
* Starts one job scheduler with the provided concurrency configuration.
|
|
1157
|
+
* @param userConfig - Configuration for the scheduler. Required field: jobs
|
|
1039
1158
|
*/
|
|
1040
1159
|
startWorkers(userConfig) {
|
|
1041
1160
|
const maxTriesReachedRetentionMs = userConfig.maxTriesReachedRetentionMs === void 0 ? DEFAULT_MAX_TRIES_REACHED_RETENTION_MS : userConfig.maxTriesReachedRetentionMs;
|
|
@@ -1043,7 +1162,6 @@ var WorkerService = class {
|
|
|
1043
1162
|
throw new Error("maxTriesReachedRetentionMs must be null, zero, or a positive number");
|
|
1044
1163
|
}
|
|
1045
1164
|
const config = {
|
|
1046
|
-
cooldownPeriod: 100,
|
|
1047
1165
|
pollInterval: 3e3,
|
|
1048
1166
|
workersCount: 4,
|
|
1049
1167
|
defaultLockTime: 30 * 1e3,
|
|
@@ -1052,8 +1170,11 @@ var WorkerService = class {
|
|
|
1052
1170
|
};
|
|
1053
1171
|
setNameToJobs(config.jobs);
|
|
1054
1172
|
const workersInstance = this.createWorkersInstanceDefinition(config);
|
|
1055
|
-
import_logger4.logger.debug("Starting
|
|
1056
|
-
this.
|
|
1173
|
+
import_logger4.logger.debug("Starting job scheduler", config);
|
|
1174
|
+
workersInstance.schedulerPromise = this.runScheduler(config, workersInstance).catch((error) => {
|
|
1175
|
+
workersInstance.running = false;
|
|
1176
|
+
import_logger4.logger.error("Job scheduler stopped after an unexpected error.", { error });
|
|
1177
|
+
});
|
|
1057
1178
|
return workersInstance;
|
|
1058
1179
|
}
|
|
1059
1180
|
};
|