@terreno/jobs 57.4.0 → 57.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +3 -3
- package/dist/activeJobExecution.js +7 -7
- package/dist/claimLock.js +12 -14
- package/dist/dispatchCompensation.js +29 -83
- package/dist/dispatchError.js +12 -28
- package/dist/jobExecutor.js +178 -297
- package/dist/jobsApp.js +124 -234
- package/dist/jobsService.js +261 -461
- package/dist/models/job.js +9 -9
- package/dist/models/jobSchedule.js +4 -4
- package/dist/retryBackoff.js +11 -14
- package/dist/routes/jobsAdmin.js +189 -370
- package/dist/routes/jobsExecute.js +41 -101
- package/dist/runners/gcpCloudTasks.js +116 -200
- package/dist/runners/mongoRunner.js +90 -184
- package/dist/runners/vercelQueues.js +105 -194
- package/dist/scheduleCron.js +18 -27
- package/dist/scheduler.js +68 -170
- package/package.json +4 -4
- package/src/models/job.ts +4 -4
- package/src/models/jobSchedule.ts +3 -2
- package/src/tests/packageExports.test.ts +19 -3
- package/tsconfig.json +6 -21
package/dist/jobsService.js
CHANGED
|
@@ -1,112 +1,38 @@
|
|
|
1
1
|
"use strict";
|
|
2
|
-
var __assign = (this && this.__assign) || function () {
|
|
3
|
-
__assign = Object.assign || function(t) {
|
|
4
|
-
for (var s, i = 1, n = arguments.length; i < n; i++) {
|
|
5
|
-
s = arguments[i];
|
|
6
|
-
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p))
|
|
7
|
-
t[p] = s[p];
|
|
8
|
-
}
|
|
9
|
-
return t;
|
|
10
|
-
};
|
|
11
|
-
return __assign.apply(this, arguments);
|
|
12
|
-
};
|
|
13
|
-
var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
|
|
14
|
-
function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
|
|
15
|
-
return new (P || (P = Promise))(function (resolve, reject) {
|
|
16
|
-
function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
|
|
17
|
-
function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
|
|
18
|
-
function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
|
|
19
|
-
step((generator = generator.apply(thisArg, _arguments || [])).next());
|
|
20
|
-
});
|
|
21
|
-
};
|
|
22
|
-
var __generator = (this && this.__generator) || function (thisArg, body) {
|
|
23
|
-
var _ = { label: 0, sent: function() { if (t[0] & 1) throw t[1]; return t[1]; }, trys: [], ops: [] }, f, y, t, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype);
|
|
24
|
-
return g.next = verb(0), g["throw"] = verb(1), g["return"] = verb(2), typeof Symbol === "function" && (g[Symbol.iterator] = function() { return this; }), g;
|
|
25
|
-
function verb(n) { return function (v) { return step([n, v]); }; }
|
|
26
|
-
function step(op) {
|
|
27
|
-
if (f) throw new TypeError("Generator is already executing.");
|
|
28
|
-
while (g && (g = 0, op[0] && (_ = 0)), _) try {
|
|
29
|
-
if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
|
|
30
|
-
if (y = 0, t) op = [op[0] & 2, t.value];
|
|
31
|
-
switch (op[0]) {
|
|
32
|
-
case 0: case 1: t = op; break;
|
|
33
|
-
case 4: _.label++; return { value: op[1], done: false };
|
|
34
|
-
case 5: _.label++; y = op[1]; op = [0]; continue;
|
|
35
|
-
case 7: op = _.ops.pop(); _.trys.pop(); continue;
|
|
36
|
-
default:
|
|
37
|
-
if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) { _ = 0; continue; }
|
|
38
|
-
if (op[0] === 3 && (!t || (op[1] > t[0] && op[1] < t[3]))) { _.label = op[1]; break; }
|
|
39
|
-
if (op[0] === 6 && _.label < t[1]) { _.label = t[1]; t = op; break; }
|
|
40
|
-
if (t && _.label < t[2]) { _.label = t[2]; _.ops.push(op); break; }
|
|
41
|
-
if (t[2]) _.ops.pop();
|
|
42
|
-
_.trys.pop(); continue;
|
|
43
|
-
}
|
|
44
|
-
op = body.call(thisArg, _);
|
|
45
|
-
} catch (e) { op = [6, e]; y = 0; } finally { f = t = 0; }
|
|
46
|
-
if (op[0] & 5) throw op[1]; return { value: op[0] ? op[1] : void 0, done: true };
|
|
47
|
-
}
|
|
48
|
-
};
|
|
49
|
-
var __values = (this && this.__values) || function(o) {
|
|
50
|
-
var s = typeof Symbol === "function" && Symbol.iterator, m = s && o[s], i = 0;
|
|
51
|
-
if (m) return m.call(o);
|
|
52
|
-
if (o && typeof o.length === "number") return {
|
|
53
|
-
next: function () {
|
|
54
|
-
if (o && i >= o.length) o = void 0;
|
|
55
|
-
return { value: o && o[i++], done: !o };
|
|
56
|
-
}
|
|
57
|
-
};
|
|
58
|
-
throw new TypeError(s ? "Object is not iterable." : "Symbol.iterator is not defined.");
|
|
59
|
-
};
|
|
60
|
-
var __read = (this && this.__read) || function (o, n) {
|
|
61
|
-
var m = typeof Symbol === "function" && o[Symbol.iterator];
|
|
62
|
-
if (!m) return o;
|
|
63
|
-
var i = m.call(o), r, ar = [], e;
|
|
64
|
-
try {
|
|
65
|
-
while ((n === void 0 || n-- > 0) && !(r = i.next()).done) ar.push(r.value);
|
|
66
|
-
}
|
|
67
|
-
catch (error) { e = { error: error }; }
|
|
68
|
-
finally {
|
|
69
|
-
try {
|
|
70
|
-
if (r && !r.done && (m = i["return"])) m.call(i);
|
|
71
|
-
}
|
|
72
|
-
finally { if (e) throw e.error; }
|
|
73
|
-
}
|
|
74
|
-
return ar;
|
|
75
|
-
};
|
|
76
2
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
77
3
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
78
4
|
};
|
|
79
5
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
80
6
|
exports.unregisterJobsService = exports.registerJobsService = exports.tryGetJobsService = exports.getJobsService = exports.JobsService = void 0;
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
this.defaultTimezone =
|
|
106
|
-
this.lockTtlMs =
|
|
107
|
-
this.runner = options
|
|
7
|
+
const api_1 = require("@terreno/api");
|
|
8
|
+
const luxon_1 = require("luxon");
|
|
9
|
+
const mongoose_1 = __importDefault(require("mongoose"));
|
|
10
|
+
const activeJobExecution_1 = require("./activeJobExecution");
|
|
11
|
+
const dispatchCompensation_1 = require("./dispatchCompensation");
|
|
12
|
+
const job_1 = require("./models/job");
|
|
13
|
+
const jobSchedule_1 = require("./models/jobSchedule");
|
|
14
|
+
const retryBackoff_1 = require("./retryBackoff");
|
|
15
|
+
const scheduleCron_1 = require("./scheduleCron");
|
|
16
|
+
const scheduler_1 = require("./scheduler");
|
|
17
|
+
const DEFAULT_MAX_ATTEMPTS = 5;
|
|
18
|
+
const DEFAULT_TIMEZONE = "UTC";
|
|
19
|
+
const UNKNOWN_JOB_NAME_TITLE = "Unknown job name";
|
|
20
|
+
const isDuplicateKeyError = (error) => Boolean(error &&
|
|
21
|
+
typeof error === "object" &&
|
|
22
|
+
"code" in error &&
|
|
23
|
+
error.code === 11_000);
|
|
24
|
+
class JobsService {
|
|
25
|
+
defaultTimezone;
|
|
26
|
+
definitions = new Map();
|
|
27
|
+
lockTtlMs;
|
|
28
|
+
runner;
|
|
29
|
+
schedulesDirty = false;
|
|
30
|
+
constructor(options) {
|
|
31
|
+
this.defaultTimezone = options?.defaultTimezone ?? DEFAULT_TIMEZONE;
|
|
32
|
+
this.lockTtlMs = options?.lockTtlMs ?? 15 * 60 * 1000;
|
|
33
|
+
this.runner = options?.runner;
|
|
108
34
|
}
|
|
109
|
-
|
|
35
|
+
define(name, definition) {
|
|
110
36
|
if (definition.schedule) {
|
|
111
37
|
(0, scheduleCron_1.validateScheduleDefinition)({
|
|
112
38
|
cron: definition.schedule.cron,
|
|
@@ -115,376 +41,250 @@ var JobsService = /** @class */ (function () {
|
|
|
115
41
|
});
|
|
116
42
|
}
|
|
117
43
|
this.definitions.set(name, definition);
|
|
118
|
-
}
|
|
119
|
-
|
|
44
|
+
}
|
|
45
|
+
getDefinition(name) {
|
|
120
46
|
return this.definitions.get(name);
|
|
121
|
-
}
|
|
122
|
-
|
|
47
|
+
}
|
|
48
|
+
getDefaultTimezone() {
|
|
123
49
|
return this.defaultTimezone;
|
|
124
|
-
}
|
|
125
|
-
|
|
50
|
+
}
|
|
51
|
+
getLockTtlMs() {
|
|
126
52
|
return this.lockTtlMs;
|
|
127
|
-
}
|
|
128
|
-
|
|
53
|
+
}
|
|
54
|
+
isSchedulesDirty() {
|
|
129
55
|
return this.schedulesDirty;
|
|
130
|
-
}
|
|
131
|
-
|
|
56
|
+
}
|
|
57
|
+
markSchedulesDirty() {
|
|
132
58
|
this.schedulesDirty = true;
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
return
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
});
|
|
149
|
-
};
|
|
150
|
-
JobsService.prototype.reconcileSchedules = function () {
|
|
151
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
152
|
-
var _a, _b, _c, name_1, definition, timezone, cron, existing, cronTimezoneChanged, update, e_1_1;
|
|
153
|
-
var e_1, _d;
|
|
154
|
-
return __generator(this, function (_e) {
|
|
155
|
-
switch (_e.label) {
|
|
156
|
-
case 0:
|
|
157
|
-
_e.trys.push([0, 8, 9, 10]);
|
|
158
|
-
_a = __values(this.definitions), _b = _a.next();
|
|
159
|
-
_e.label = 1;
|
|
160
|
-
case 1:
|
|
161
|
-
if (!!_b.done) return [3 /*break*/, 7];
|
|
162
|
-
_c = __read(_b.value, 2), name_1 = _c[0], definition = _c[1];
|
|
163
|
-
if (!definition.schedule) {
|
|
164
|
-
return [3 /*break*/, 6];
|
|
165
|
-
}
|
|
166
|
-
timezone = (0, scheduleCron_1.resolveScheduleTimezone)({
|
|
167
|
-
defaultTimezone: this.defaultTimezone,
|
|
168
|
-
scheduleTimezone: definition.schedule.timezone,
|
|
169
|
-
});
|
|
170
|
-
cron = definition.schedule.cron;
|
|
171
|
-
return [4 /*yield*/, jobSchedule_1.JobSchedule.findOneOrNone({ name: name_1 })];
|
|
172
|
-
case 2:
|
|
173
|
-
existing = _e.sent();
|
|
174
|
-
if (!!existing) return [3 /*break*/, 4];
|
|
175
|
-
return [4 /*yield*/, jobSchedule_1.JobSchedule.findOneAndUpdate({ name: name_1 }, {
|
|
176
|
-
$setOnInsert: {
|
|
177
|
-
cron: cron,
|
|
178
|
-
enabled: true,
|
|
179
|
-
handlerName: name_1,
|
|
180
|
-
nextRunAt: (0, scheduleCron_1.computeInitialNextRunAt)(cron, timezone),
|
|
181
|
-
timezone: timezone,
|
|
182
|
-
},
|
|
183
|
-
}, { upsert: true })];
|
|
184
|
-
case 3:
|
|
185
|
-
_e.sent();
|
|
186
|
-
return [3 /*break*/, 6];
|
|
187
|
-
case 4:
|
|
188
|
-
cronTimezoneChanged = existing.cron !== cron || existing.timezone !== timezone;
|
|
189
|
-
update = {
|
|
190
|
-
cron: cron,
|
|
191
|
-
handlerName: name_1,
|
|
192
|
-
timezone: timezone,
|
|
193
|
-
};
|
|
194
|
-
if (cronTimezoneChanged) {
|
|
195
|
-
update.nextRunAt = (0, scheduleCron_1.computeInitialNextRunAt)(cron, timezone);
|
|
196
|
-
}
|
|
197
|
-
return [4 /*yield*/, jobSchedule_1.JobSchedule.updateOne({ _id: existing._id }, { $set: update })];
|
|
198
|
-
case 5:
|
|
199
|
-
_e.sent();
|
|
200
|
-
_e.label = 6;
|
|
201
|
-
case 6:
|
|
202
|
-
_b = _a.next();
|
|
203
|
-
return [3 /*break*/, 1];
|
|
204
|
-
case 7: return [3 /*break*/, 10];
|
|
205
|
-
case 8:
|
|
206
|
-
e_1_1 = _e.sent();
|
|
207
|
-
e_1 = { error: e_1_1 };
|
|
208
|
-
return [3 /*break*/, 10];
|
|
209
|
-
case 9:
|
|
210
|
-
try {
|
|
211
|
-
if (_b && !_b.done && (_d = _a.return)) _d.call(_a);
|
|
212
|
-
}
|
|
213
|
-
finally { if (e_1) throw e_1.error; }
|
|
214
|
-
return [7 /*endfinally*/];
|
|
215
|
-
case 10:
|
|
216
|
-
this.schedulesDirty = false;
|
|
217
|
-
return [2 /*return*/];
|
|
218
|
-
}
|
|
219
|
-
});
|
|
220
|
-
});
|
|
221
|
-
};
|
|
222
|
-
JobsService.prototype.tickSchedules = function (now) {
|
|
223
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
224
|
-
return __generator(this, function (_a) {
|
|
225
|
-
switch (_a.label) {
|
|
226
|
-
case 0: return [4 /*yield*/, (0, scheduler_1.tickDueSchedules)({
|
|
227
|
-
jobsService: this,
|
|
228
|
-
now: now,
|
|
229
|
-
})];
|
|
230
|
-
case 1:
|
|
231
|
-
_a.sent();
|
|
232
|
-
return [2 /*return*/];
|
|
233
|
-
}
|
|
234
|
-
});
|
|
235
|
-
});
|
|
236
|
-
};
|
|
237
|
-
JobsService.prototype.enqueue = function (params) {
|
|
238
|
-
return __awaiter(this, void 0, void 0, function () {
|
|
239
|
-
var definition, runAt, maxAttempts, backoffMs, maxBackoffMs, idempotencyKey, existing, createPayload, job, error_1;
|
|
240
|
-
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
241
|
-
return __generator(this, function (_j) {
|
|
242
|
-
switch (_j.label) {
|
|
243
|
-
case 0:
|
|
244
|
-
definition = this.definitions.get(params.name);
|
|
245
|
-
if (!definition) {
|
|
246
|
-
throw new api_1.APIError({ status: 400, title: UNKNOWN_JOB_NAME_TITLE });
|
|
247
|
-
}
|
|
248
|
-
runAt = (_a = params.runAt) !== null && _a !== void 0 ? _a : luxon_1.DateTime.utc().toJSDate();
|
|
249
|
-
maxAttempts = (_c = (_b = definition.retry) === null || _b === void 0 ? void 0 : _b.maxAttempts) !== null && _c !== void 0 ? _c : DEFAULT_MAX_ATTEMPTS;
|
|
250
|
-
backoffMs = (_e = (_d = definition.retry) === null || _d === void 0 ? void 0 : _d.backoffMs) !== null && _e !== void 0 ? _e : retryBackoff_1.DEFAULT_BACKOFF_MS;
|
|
251
|
-
maxBackoffMs = (_g = (_f = definition.retry) === null || _f === void 0 ? void 0 : _f.maxBackoffMs) !== null && _g !== void 0 ? _g : retryBackoff_1.DEFAULT_MAX_BACKOFF_MS;
|
|
252
|
-
idempotencyKey = (_h = params.idempotencyKey) === null || _h === void 0 ? void 0 : _h.trim();
|
|
253
|
-
if (!idempotencyKey) return [3 /*break*/, 2];
|
|
254
|
-
return [4 /*yield*/, job_1.Job.findOneOrNone({
|
|
255
|
-
idempotencyKey: idempotencyKey,
|
|
256
|
-
name: params.name,
|
|
257
|
-
})];
|
|
258
|
-
case 1:
|
|
259
|
-
existing = _j.sent();
|
|
260
|
-
if (existing) {
|
|
261
|
-
return [2 /*return*/, existing];
|
|
262
|
-
}
|
|
263
|
-
_j.label = 2;
|
|
264
|
-
case 2:
|
|
265
|
-
createPayload = __assign(__assign({ attemptCount: 0, backoffMs: backoffMs, maxAttempts: maxAttempts, maxBackoffMs: maxBackoffMs, name: params.name, payload: params.payload, payloadRedacted: false, runAt: runAt, status: "pending" }, (idempotencyKey ? { idempotencyKey: idempotencyKey } : {})), (params.scheduleId ? { scheduleId: new mongoose_1.default.Types.ObjectId(params.scheduleId) } : {}));
|
|
266
|
-
_j.label = 3;
|
|
267
|
-
case 3:
|
|
268
|
-
_j.trys.push([3, 5, , 6]);
|
|
269
|
-
return [4 /*yield*/, job_1.Job.create(createPayload)];
|
|
270
|
-
case 4:
|
|
271
|
-
job = _j.sent();
|
|
272
|
-
return [3 /*break*/, 6];
|
|
273
|
-
case 5:
|
|
274
|
-
error_1 = _j.sent();
|
|
275
|
-
if (!idempotencyKey || !isDuplicateKeyError(error_1)) {
|
|
276
|
-
throw error_1;
|
|
277
|
-
}
|
|
278
|
-
return [2 /*return*/, job_1.Job.findExactlyOne({
|
|
279
|
-
idempotencyKey: idempotencyKey,
|
|
280
|
-
name: params.name,
|
|
281
|
-
})];
|
|
282
|
-
case 6:
|
|
283
|
-
if (!this.runner) return [3 /*break*/, 8];
|
|
284
|
-
return [4 /*yield*/, (0, dispatchCompensation_1.dispatchJobToRunner)(this.runner, job)];
|
|
285
|
-
case 7:
|
|
286
|
-
_j.sent();
|
|
287
|
-
_j.label = 8;
|
|
288
|
-
case 8: return [2 /*return*/, job];
|
|
289
|
-
}
|
|
59
|
+
}
|
|
60
|
+
async reconcileSchedulesIfDirty() {
|
|
61
|
+
if (!this.schedulesDirty) {
|
|
62
|
+
return;
|
|
63
|
+
}
|
|
64
|
+
await this.reconcileSchedules();
|
|
65
|
+
}
|
|
66
|
+
async reconcileSchedules() {
|
|
67
|
+
for (const [name, definition] of this.definitions) {
|
|
68
|
+
if (!definition.schedule) {
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
const timezone = (0, scheduleCron_1.resolveScheduleTimezone)({
|
|
72
|
+
defaultTimezone: this.defaultTimezone,
|
|
73
|
+
scheduleTimezone: definition.schedule.timezone,
|
|
290
74
|
});
|
|
75
|
+
const cron = definition.schedule.cron;
|
|
76
|
+
const existing = await jobSchedule_1.JobSchedule.findOneOrNone({ name });
|
|
77
|
+
if (!existing) {
|
|
78
|
+
await jobSchedule_1.JobSchedule.findOneAndUpdate({ name }, {
|
|
79
|
+
$setOnInsert: {
|
|
80
|
+
cron,
|
|
81
|
+
enabled: true,
|
|
82
|
+
handlerName: name,
|
|
83
|
+
nextRunAt: (0, scheduleCron_1.computeInitialNextRunAt)(cron, timezone),
|
|
84
|
+
timezone,
|
|
85
|
+
},
|
|
86
|
+
}, { upsert: true });
|
|
87
|
+
continue;
|
|
88
|
+
}
|
|
89
|
+
const cronTimezoneChanged = existing.cron !== cron || existing.timezone !== timezone;
|
|
90
|
+
const update = {
|
|
91
|
+
cron,
|
|
92
|
+
handlerName: name,
|
|
93
|
+
timezone,
|
|
94
|
+
};
|
|
95
|
+
if (cronTimezoneChanged) {
|
|
96
|
+
update.nextRunAt = (0, scheduleCron_1.computeInitialNextRunAt)(cron, timezone);
|
|
97
|
+
}
|
|
98
|
+
await jobSchedule_1.JobSchedule.updateOne({ _id: existing._id }, { $set: update });
|
|
99
|
+
}
|
|
100
|
+
this.schedulesDirty = false;
|
|
101
|
+
}
|
|
102
|
+
async tickSchedules(now) {
|
|
103
|
+
await (0, scheduler_1.tickDueSchedules)({
|
|
104
|
+
jobsService: this,
|
|
105
|
+
now,
|
|
291
106
|
});
|
|
292
|
-
}
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
296
|
-
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
}
|
|
308
|
-
if (original.retriedById) {
|
|
309
|
-
throw new api_1.APIError({ status: 409, title: "Job already retried" });
|
|
310
|
-
}
|
|
311
|
-
if (original.status !== "dead" && original.status !== "failed") {
|
|
312
|
-
throw new api_1.APIError({
|
|
313
|
-
detail: "Status is ".concat(original.status),
|
|
314
|
-
status: 400,
|
|
315
|
-
title: "Job cannot be retried",
|
|
316
|
-
});
|
|
317
|
-
}
|
|
318
|
-
definition = this.definitions.get(original.name);
|
|
319
|
-
if (!definition) {
|
|
320
|
-
throw new api_1.APIError({ status: 400, title: UNKNOWN_JOB_NAME_TITLE });
|
|
321
|
-
}
|
|
322
|
-
now = luxon_1.DateTime.utc().toJSDate();
|
|
323
|
-
return [4 /*yield*/, job_1.Job.create({
|
|
324
|
-
attemptCount: 0,
|
|
325
|
-
attempts: [],
|
|
326
|
-
backoffMs: original.backoffMs,
|
|
327
|
-
maxAttempts: original.maxAttempts,
|
|
328
|
-
maxBackoffMs: original.maxBackoffMs,
|
|
329
|
-
name: original.name,
|
|
330
|
-
payload: original.payload,
|
|
331
|
-
payloadRedacted: original.payloadRedacted,
|
|
332
|
-
retriedFromId: original._id,
|
|
333
|
-
runAt: now,
|
|
334
|
-
scheduleId: original.scheduleId,
|
|
335
|
-
status: "pending",
|
|
336
|
-
})];
|
|
337
|
-
case 2:
|
|
338
|
-
retry = _a.sent();
|
|
339
|
-
return [4 /*yield*/, job_1.Job.findOneAndUpdate({
|
|
340
|
-
_id: original._id,
|
|
341
|
-
retriedById: { $exists: false },
|
|
342
|
-
status: { $in: ["dead", "failed"] },
|
|
343
|
-
}, { $set: { retriedById: retry._id } }, { returnDocument: "after" })];
|
|
344
|
-
case 3:
|
|
345
|
-
linked = _a.sent();
|
|
346
|
-
if (!!linked) return [3 /*break*/, 6];
|
|
347
|
-
return [4 /*yield*/, job_1.Job.deleteOne({ _id: retry._id })];
|
|
348
|
-
case 4:
|
|
349
|
-
_a.sent();
|
|
350
|
-
return [4 /*yield*/, job_1.Job.findOneOrNone({ _id: original._id })];
|
|
351
|
-
case 5:
|
|
352
|
-
current = _a.sent();
|
|
353
|
-
if (current === null || current === void 0 ? void 0 : current.retriedById) {
|
|
354
|
-
throw new api_1.APIError({ status: 409, title: "Job already retried" });
|
|
355
|
-
}
|
|
356
|
-
throw new api_1.APIError({ status: 409, title: "Job state changed" });
|
|
357
|
-
case 6:
|
|
358
|
-
if (!this.runner) return [3 /*break*/, 8];
|
|
359
|
-
return [4 /*yield*/, (0, dispatchCompensation_1.dispatchJobToRunner)(this.runner, retry)];
|
|
360
|
-
case 7:
|
|
361
|
-
_a.sent();
|
|
362
|
-
_a.label = 8;
|
|
363
|
-
case 8: return [2 /*return*/, { original: linked, retry: retry }];
|
|
364
|
-
}
|
|
107
|
+
}
|
|
108
|
+
async enqueue(params) {
|
|
109
|
+
const definition = this.definitions.get(params.name);
|
|
110
|
+
if (!definition) {
|
|
111
|
+
throw new api_1.APIError({ status: 400, title: UNKNOWN_JOB_NAME_TITLE });
|
|
112
|
+
}
|
|
113
|
+
const runAt = params.runAt ?? luxon_1.DateTime.utc().toJSDate();
|
|
114
|
+
const maxAttempts = definition.retry?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
|
|
115
|
+
const backoffMs = definition.retry?.backoffMs ?? retryBackoff_1.DEFAULT_BACKOFF_MS;
|
|
116
|
+
const maxBackoffMs = definition.retry?.maxBackoffMs ?? retryBackoff_1.DEFAULT_MAX_BACKOFF_MS;
|
|
117
|
+
const idempotencyKey = params.idempotencyKey?.trim();
|
|
118
|
+
if (idempotencyKey) {
|
|
119
|
+
const existing = await job_1.Job.findOneOrNone({
|
|
120
|
+
idempotencyKey,
|
|
121
|
+
name: params.name,
|
|
365
122
|
});
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
}
|
|
395
|
-
if (existing.retriedById) {
|
|
396
|
-
throw new api_1.APIError({ status: 409, title: "Job already retried" });
|
|
397
|
-
}
|
|
398
|
-
throw new api_1.APIError({
|
|
399
|
-
detail: "Status is ".concat(existing.status),
|
|
400
|
-
status: 400,
|
|
401
|
-
title: "Job cannot be requeued",
|
|
402
|
-
});
|
|
403
|
-
case 3:
|
|
404
|
-
if (!this.runner) return [3 /*break*/, 5];
|
|
405
|
-
return [4 /*yield*/, (0, dispatchCompensation_1.dispatchJobToRunner)(this.runner, requeued)];
|
|
406
|
-
case 4:
|
|
407
|
-
_a.sent();
|
|
408
|
-
_a.label = 5;
|
|
409
|
-
case 5: return [2 /*return*/, requeued];
|
|
410
|
-
}
|
|
123
|
+
if (existing) {
|
|
124
|
+
return existing;
|
|
125
|
+
}
|
|
126
|
+
}
|
|
127
|
+
const createPayload = {
|
|
128
|
+
attemptCount: 0,
|
|
129
|
+
backoffMs,
|
|
130
|
+
maxAttempts,
|
|
131
|
+
maxBackoffMs,
|
|
132
|
+
name: params.name,
|
|
133
|
+
payload: params.payload,
|
|
134
|
+
payloadRedacted: false,
|
|
135
|
+
runAt,
|
|
136
|
+
status: "pending",
|
|
137
|
+
...(idempotencyKey ? { idempotencyKey } : {}),
|
|
138
|
+
...(params.scheduleId ? { scheduleId: new mongoose_1.default.Types.ObjectId(params.scheduleId) } : {}),
|
|
139
|
+
};
|
|
140
|
+
let job;
|
|
141
|
+
try {
|
|
142
|
+
job = await job_1.Job.create(createPayload);
|
|
143
|
+
}
|
|
144
|
+
catch (error) {
|
|
145
|
+
if (!idempotencyKey || !isDuplicateKeyError(error)) {
|
|
146
|
+
throw error;
|
|
147
|
+
}
|
|
148
|
+
return job_1.Job.findExactlyOne({
|
|
149
|
+
idempotencyKey,
|
|
150
|
+
name: params.name,
|
|
411
151
|
});
|
|
412
|
-
}
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
case 2:
|
|
435
|
-
existing = _a.sent();
|
|
436
|
-
if (!existing) {
|
|
437
|
-
throw new api_1.NotFoundError("Job not found");
|
|
438
|
-
}
|
|
439
|
-
throw new api_1.APIError({
|
|
440
|
-
detail: "Status is ".concat(existing.status),
|
|
441
|
-
status: 400,
|
|
442
|
-
title: "Job cannot be cancelled",
|
|
443
|
-
});
|
|
444
|
-
case 3:
|
|
445
|
-
(0, activeJobExecution_1.abortActiveJobExecution)(jobId);
|
|
446
|
-
return [2 /*return*/, cancelled];
|
|
447
|
-
}
|
|
152
|
+
}
|
|
153
|
+
if (this.runner) {
|
|
154
|
+
await (0, dispatchCompensation_1.dispatchJobToRunner)(this.runner, job);
|
|
155
|
+
}
|
|
156
|
+
return job;
|
|
157
|
+
}
|
|
158
|
+
async adminRetryJob(jobId) {
|
|
159
|
+
if (!mongoose_1.default.isValidObjectId(jobId)) {
|
|
160
|
+
throw new api_1.NotFoundError("Job not found");
|
|
161
|
+
}
|
|
162
|
+
const original = await job_1.Job.findOneOrNone({ _id: jobId });
|
|
163
|
+
if (!original) {
|
|
164
|
+
throw new api_1.NotFoundError("Job not found");
|
|
165
|
+
}
|
|
166
|
+
if (original.retriedById) {
|
|
167
|
+
throw new api_1.APIError({ status: 409, title: "Job already retried" });
|
|
168
|
+
}
|
|
169
|
+
if (original.status !== "dead" && original.status !== "failed") {
|
|
170
|
+
throw new api_1.APIError({
|
|
171
|
+
detail: `Status is ${original.status}`,
|
|
172
|
+
status: 400,
|
|
173
|
+
title: "Job cannot be retried",
|
|
448
174
|
});
|
|
175
|
+
}
|
|
176
|
+
const definition = this.definitions.get(original.name);
|
|
177
|
+
if (!definition) {
|
|
178
|
+
throw new api_1.APIError({ status: 400, title: UNKNOWN_JOB_NAME_TITLE });
|
|
179
|
+
}
|
|
180
|
+
const now = luxon_1.DateTime.utc().toJSDate();
|
|
181
|
+
const retry = await job_1.Job.create({
|
|
182
|
+
attemptCount: 0,
|
|
183
|
+
attempts: [],
|
|
184
|
+
backoffMs: original.backoffMs,
|
|
185
|
+
maxAttempts: original.maxAttempts,
|
|
186
|
+
maxBackoffMs: original.maxBackoffMs,
|
|
187
|
+
name: original.name,
|
|
188
|
+
payload: original.payload,
|
|
189
|
+
payloadRedacted: original.payloadRedacted,
|
|
190
|
+
retriedFromId: original._id,
|
|
191
|
+
runAt: now,
|
|
192
|
+
scheduleId: original.scheduleId,
|
|
193
|
+
status: "pending",
|
|
449
194
|
});
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
195
|
+
const linked = await job_1.Job.findOneAndUpdate({
|
|
196
|
+
_id: original._id,
|
|
197
|
+
retriedById: { $exists: false },
|
|
198
|
+
status: { $in: ["dead", "failed"] },
|
|
199
|
+
}, { $set: { retriedById: retry._id } }, { returnDocument: "after" });
|
|
200
|
+
if (!linked) {
|
|
201
|
+
await job_1.Job.deleteOne({ _id: retry._id });
|
|
202
|
+
const current = await job_1.Job.findOneOrNone({ _id: original._id });
|
|
203
|
+
if (current?.retriedById) {
|
|
204
|
+
throw new api_1.APIError({ status: 409, title: "Job already retried" });
|
|
205
|
+
}
|
|
206
|
+
throw new api_1.APIError({ status: 409, title: "Job state changed" });
|
|
207
|
+
}
|
|
208
|
+
if (this.runner) {
|
|
209
|
+
await (0, dispatchCompensation_1.dispatchJobToRunner)(this.runner, retry);
|
|
210
|
+
}
|
|
211
|
+
return { original: linked, retry };
|
|
212
|
+
}
|
|
213
|
+
async adminRequeueJob(jobId) {
|
|
214
|
+
if (!mongoose_1.default.isValidObjectId(jobId)) {
|
|
215
|
+
throw new api_1.NotFoundError("Job not found");
|
|
216
|
+
}
|
|
217
|
+
const now = luxon_1.DateTime.utc().toJSDate();
|
|
218
|
+
const requeued = await job_1.Job.findOneAndUpdate({
|
|
219
|
+
_id: jobId,
|
|
220
|
+
retriedById: { $exists: false },
|
|
221
|
+
status: { $in: ["cancelled", "dead", "failed"] },
|
|
222
|
+
}, {
|
|
223
|
+
$set: { runAt: now, status: "pending" },
|
|
224
|
+
$unset: { lockedAt: "", lockedBy: "" },
|
|
225
|
+
}, { returnDocument: "after" });
|
|
226
|
+
if (!requeued) {
|
|
227
|
+
const existing = await job_1.Job.findOneOrNone({ _id: jobId });
|
|
228
|
+
if (!existing) {
|
|
229
|
+
throw new api_1.NotFoundError("Job not found");
|
|
230
|
+
}
|
|
231
|
+
if (existing.retriedById) {
|
|
232
|
+
throw new api_1.APIError({ status: 409, title: "Job already retried" });
|
|
233
|
+
}
|
|
234
|
+
throw new api_1.APIError({
|
|
235
|
+
detail: `Status is ${existing.status}`,
|
|
236
|
+
status: 400,
|
|
237
|
+
title: "Job cannot be requeued",
|
|
464
238
|
});
|
|
465
|
-
}
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
239
|
+
}
|
|
240
|
+
if (this.runner) {
|
|
241
|
+
await (0, dispatchCompensation_1.dispatchJobToRunner)(this.runner, requeued);
|
|
242
|
+
}
|
|
243
|
+
return requeued;
|
|
244
|
+
}
|
|
245
|
+
async adminCancelJob(jobId) {
|
|
246
|
+
if (!mongoose_1.default.isValidObjectId(jobId)) {
|
|
247
|
+
throw new api_1.NotFoundError("Job not found");
|
|
248
|
+
}
|
|
249
|
+
const cancelled = await job_1.Job.findOneAndUpdate({
|
|
250
|
+
_id: jobId,
|
|
251
|
+
status: { $in: ["pending", "running", "scheduled"] },
|
|
252
|
+
}, {
|
|
253
|
+
$set: { status: "cancelled" },
|
|
254
|
+
$unset: { lockedAt: "", lockedBy: "" },
|
|
255
|
+
}, { returnDocument: "after" });
|
|
256
|
+
if (!cancelled) {
|
|
257
|
+
const existing = await job_1.Job.findOneOrNone({ _id: jobId });
|
|
258
|
+
if (!existing) {
|
|
259
|
+
throw new api_1.NotFoundError("Job not found");
|
|
260
|
+
}
|
|
261
|
+
throw new api_1.APIError({
|
|
262
|
+
detail: `Status is ${existing.status}`,
|
|
263
|
+
status: 400,
|
|
264
|
+
title: "Job cannot be cancelled",
|
|
480
265
|
});
|
|
481
|
-
}
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
}
|
|
266
|
+
}
|
|
267
|
+
(0, activeJobExecution_1.abortActiveJobExecution)(jobId);
|
|
268
|
+
return cancelled;
|
|
269
|
+
}
|
|
270
|
+
async pauseSchedule(name) {
|
|
271
|
+
const updated = await jobSchedule_1.JobSchedule.findOneAndUpdate({ name }, { $set: { enabled: false } }, { returnDocument: "after" });
|
|
272
|
+
if (!updated) {
|
|
273
|
+
throw new api_1.NotFoundError("Schedule not found");
|
|
274
|
+
}
|
|
275
|
+
return updated;
|
|
276
|
+
}
|
|
277
|
+
async resumeSchedule(name) {
|
|
278
|
+
const updated = await jobSchedule_1.JobSchedule.findOneAndUpdate({ name }, { $set: { enabled: true } }, { returnDocument: "after" });
|
|
279
|
+
if (!updated) {
|
|
280
|
+
throw new api_1.NotFoundError("Schedule not found");
|
|
281
|
+
}
|
|
282
|
+
return updated;
|
|
283
|
+
}
|
|
284
|
+
}
|
|
485
285
|
exports.JobsService = JobsService;
|
|
486
|
-
|
|
487
|
-
|
|
286
|
+
let registeredJobsService;
|
|
287
|
+
const getJobsService = () => {
|
|
488
288
|
if (!registeredJobsService) {
|
|
489
289
|
throw new api_1.APIError({ status: 500, title: "JobsApp is not registered" });
|
|
490
290
|
}
|
|
@@ -492,14 +292,14 @@ var getJobsService = function () {
|
|
|
492
292
|
};
|
|
493
293
|
exports.getJobsService = getJobsService;
|
|
494
294
|
/** Returns the registered jobs service when `JobsApp` is mounted; otherwise `undefined`. */
|
|
495
|
-
|
|
295
|
+
const tryGetJobsService = () => registeredJobsService;
|
|
496
296
|
exports.tryGetJobsService = tryGetJobsService;
|
|
497
|
-
|
|
297
|
+
const registerJobsService = (service) => {
|
|
498
298
|
registeredJobsService = service;
|
|
499
299
|
};
|
|
500
300
|
exports.registerJobsService = registerJobsService;
|
|
501
301
|
/** Clears the process-wide jobs service. Tests use this so AdminApp falls back to in-process runs. */
|
|
502
|
-
|
|
302
|
+
const unregisterJobsService = () => {
|
|
503
303
|
registeredJobsService = undefined;
|
|
504
304
|
};
|
|
505
305
|
exports.unregisterJobsService = unregisterJobsService;
|