@terreno/jobs 57.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +56 -0
- package/biome.jsonc +20 -0
- package/bunfig.toml +5 -0
- package/dist/activeJobExecution.d.ts +3 -0
- package/dist/activeJobExecution.js +29 -0
- package/dist/claimLock.d.ts +16 -0
- package/dist/claimLock.js +34 -0
- package/dist/dispatchCompensation.d.ts +3 -0
- package/dist/dispatchCompensation.js +89 -0
- package/dist/dispatchError.d.ts +10 -0
- package/dist/dispatchError.js +33 -0
- package/dist/index.d.ts +9 -0
- package/dist/index.js +15 -0
- package/dist/jobExecutor.d.ts +34 -0
- package/dist/jobExecutor.js +328 -0
- package/dist/jobsApp.d.ts +61 -0
- package/dist/jobsApp.js +308 -0
- package/dist/jobsService.d.ts +38 -0
- package/dist/jobsService.js +505 -0
- package/dist/modelTypes.d.ts +52 -0
- package/dist/modelTypes.js +4 -0
- package/dist/models/job.d.ts +2 -0
- package/dist/models/job.js +124 -0
- package/dist/models/jobSchedule.d.ts +2 -0
- package/dist/models/jobSchedule.js +48 -0
- package/dist/retryBackoff.d.ts +17 -0
- package/dist/retryBackoff.js +22 -0
- package/dist/routes/jobsAdmin.d.ts +15 -0
- package/dist/routes/jobsAdmin.js +596 -0
- package/dist/routes/jobsExecute.d.ts +12 -0
- package/dist/routes/jobsExecute.js +119 -0
- package/dist/runners/gcpCloudTasks.d.ts +78 -0
- package/dist/runners/gcpCloudTasks.js +256 -0
- package/dist/runners/mongoRunner.d.ts +14 -0
- package/dist/runners/mongoRunner.js +204 -0
- package/dist/runners/vercelQueues.d.ts +111 -0
- package/dist/runners/vercelQueues.js +326 -0
- package/dist/scheduleCron.d.ts +15 -0
- package/dist/scheduleCron.js +61 -0
- package/dist/scheduler.d.ts +8 -0
- package/dist/scheduler.js +178 -0
- package/dist/types.d.ts +64 -0
- package/dist/types.js +2 -0
- package/package.json +82 -0
- package/src/activeJobExecution.ts +28 -0
- package/src/claimLock.ts +37 -0
- package/src/dispatchCompensation.ts +39 -0
- package/src/dispatchError.ts +14 -0
- package/src/index.ts +24 -0
- package/src/jobExecutor.ts +333 -0
- package/src/jobsApp.ts +255 -0
- package/src/jobsService.ts +395 -0
- package/src/modelTypes.ts +78 -0
- package/src/models/job.ts +129 -0
- package/src/models/jobSchedule.ts +57 -0
- package/src/retryBackoff.ts +34 -0
- package/src/routes/jobsAdmin.ts +562 -0
- package/src/routes/jobsExecute.ts +95 -0
- package/src/runners/gcpCloudTasks.ts +290 -0
- package/src/runners/mongoRunner.ts +135 -0
- package/src/runners/vercelQueues.ts +394 -0
- package/src/scheduleCron.ts +65 -0
- package/src/scheduler.ts +112 -0
- package/src/tests/bunSetup.ts +9 -0
- package/src/tests/gcpCloudTasksRunner.test.ts +349 -0
- package/src/tests/jobModel.test.ts +62 -0
- package/src/tests/jobScheduleModel.test.ts +44 -0
- package/src/tests/jobsAdmin.test.ts +728 -0
- package/src/tests/jobsApp.test.ts +113 -0
- package/src/tests/jobsCustomRunner.test.ts +367 -0
- package/src/tests/jobsDelayIdempotency.test.ts +255 -0
- package/src/tests/jobsExecute.test.ts +640 -0
- package/src/tests/jobsRetry.test.ts +420 -0
- package/src/tests/jobsSchedule.test.ts +505 -0
- package/src/tests/jobsTracer.test.ts +165 -0
- package/src/tests/jobsWorkerLifecycle.test.ts +509 -0
- package/src/tests/packageExports.test.ts +24 -0
- package/src/tests/retryBackoff.test.ts +76 -0
- package/src/tests/scheduleCron.test.ts +87 -0
- package/src/tests/vercelQueuesRunner.test.ts +524 -0
- package/src/types.ts +74 -0
- package/tsconfig.json +33 -0
|
@@ -0,0 +1,505 @@
|
|
|
1
|
+
import {beforeEach, describe, it} from "bun:test";
|
|
2
|
+
import {isAPIError, TerrenoApp, type UserModel as UserModelType} from "@terreno/api";
|
|
3
|
+
import {setupDb, UserModel} from "@terreno/api/testing";
|
|
4
|
+
import {assert} from "chai";
|
|
5
|
+
import {DateTime} from "luxon";
|
|
6
|
+
|
|
7
|
+
import {JobsApp} from "../jobsApp";
|
|
8
|
+
import {getJobsService, JobsService} from "../jobsService";
|
|
9
|
+
import {Job} from "../models/job";
|
|
10
|
+
import {JobSchedule} from "../models/jobSchedule";
|
|
11
|
+
import {buildScheduleRunIdempotencyKey, tickDueSchedules} from "../scheduler";
|
|
12
|
+
|
|
13
|
+
const typedUserModel = UserModel as unknown as UserModelType;
|
|
14
|
+
|
|
15
|
+
const waitUntil = async (
|
|
16
|
+
predicate: () => Promise<boolean>,
|
|
17
|
+
options?: {intervalMs?: number; timeoutMs?: number}
|
|
18
|
+
): Promise<void> => {
|
|
19
|
+
const intervalMs = options?.intervalMs ?? 25;
|
|
20
|
+
const timeoutMs = options?.timeoutMs ?? 5_000;
|
|
21
|
+
const deadline = DateTime.utc().plus({milliseconds: timeoutMs});
|
|
22
|
+
|
|
23
|
+
while (DateTime.utc() < deadline) {
|
|
24
|
+
if (await predicate()) {
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
await Bun.sleep(intervalMs);
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
throw new Error("waitUntil timed out");
|
|
31
|
+
};
|
|
32
|
+
|
|
33
|
+
const registerJobsApp = (jobsApp: JobsApp): void => {
|
|
34
|
+
new TerrenoApp({
|
|
35
|
+
skipListen: true,
|
|
36
|
+
userModel: typedUserModel,
|
|
37
|
+
})
|
|
38
|
+
.register(jobsApp)
|
|
39
|
+
.build();
|
|
40
|
+
};
|
|
41
|
+
|
|
42
|
+
const captureError = (operation: () => void): unknown => {
|
|
43
|
+
try {
|
|
44
|
+
operation();
|
|
45
|
+
return undefined;
|
|
46
|
+
} catch (error: unknown) {
|
|
47
|
+
return error;
|
|
48
|
+
}
|
|
49
|
+
};
|
|
50
|
+
|
|
51
|
+
const startScheduledWorker = async (
|
|
52
|
+
jobsApp: JobsApp,
|
|
53
|
+
name: string,
|
|
54
|
+
handler: () => Promise<void>,
|
|
55
|
+
schedule: {cron: string; timezone?: string}
|
|
56
|
+
): Promise<void> => {
|
|
57
|
+
jobsApp.define(name, {handler, schedule});
|
|
58
|
+
registerJobsApp(jobsApp);
|
|
59
|
+
await jobsApp.startWorker();
|
|
60
|
+
};
|
|
61
|
+
|
|
62
|
+
describe("jobs recurring schedules", () => {
|
|
63
|
+
beforeEach(async (): Promise<void> => {
|
|
64
|
+
await setupDb();
|
|
65
|
+
await Job.deleteMany({});
|
|
66
|
+
await JobSchedule.deleteMany({});
|
|
67
|
+
await Job.syncIndexes();
|
|
68
|
+
await JobSchedule.syncIndexes();
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
it("rejects invalid cron when defining a scheduled job", (): void => {
|
|
72
|
+
const jobsApp = new JobsApp();
|
|
73
|
+
const error = captureError(() =>
|
|
74
|
+
jobsApp.define("bad-cron", {
|
|
75
|
+
handler: async () => {},
|
|
76
|
+
schedule: {cron: "not-a-cron"},
|
|
77
|
+
})
|
|
78
|
+
);
|
|
79
|
+
|
|
80
|
+
assert.isTrue(isAPIError(error));
|
|
81
|
+
assert.equal((error as {status: number}).status, 400);
|
|
82
|
+
});
|
|
83
|
+
|
|
84
|
+
it("rejects invalid timezone when defining a scheduled job", (): void => {
|
|
85
|
+
const jobsApp = new JobsApp();
|
|
86
|
+
const error = captureError(() =>
|
|
87
|
+
jobsApp.define("bad-timezone", {
|
|
88
|
+
handler: async () => {},
|
|
89
|
+
schedule: {cron: "0 9 * * *", timezone: "Not/A_Zone"},
|
|
90
|
+
})
|
|
91
|
+
);
|
|
92
|
+
|
|
93
|
+
assert.isTrue(isAPIError(error));
|
|
94
|
+
assert.equal((error as {status: number}).status, 400);
|
|
95
|
+
});
|
|
96
|
+
|
|
97
|
+
it("upserts JobSchedule on startWorker with default UTC timezone", async (): Promise<void> => {
|
|
98
|
+
const jobsApp = new JobsApp({pollIntervalMs: 25});
|
|
99
|
+
jobsApp.define("utc-daily", {
|
|
100
|
+
handler: async () => {},
|
|
101
|
+
schedule: {cron: "0 9 * * *"},
|
|
102
|
+
});
|
|
103
|
+
registerJobsApp(jobsApp);
|
|
104
|
+
|
|
105
|
+
await jobsApp.startWorker();
|
|
106
|
+
await jobsApp.stopWorker();
|
|
107
|
+
|
|
108
|
+
const schedule = await JobSchedule.findExactlyOne({name: "utc-daily"});
|
|
109
|
+
assert.equal(schedule.timezone, "UTC");
|
|
110
|
+
assert.equal(schedule.handlerName, "utc-daily");
|
|
111
|
+
assert.equal(schedule.enabled, true);
|
|
112
|
+
assert.isAbove(schedule.nextRunAt.getTime(), DateTime.utc().toMillis() - 1_000);
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
it("uses JobsApp timezone when the schedule omits timezone", async (): Promise<void> => {
|
|
116
|
+
const jobsApp = new JobsApp({pollIntervalMs: 25, timezone: "America/New_York"});
|
|
117
|
+
jobsApp.define("app-tz-daily", {
|
|
118
|
+
handler: async () => {},
|
|
119
|
+
schedule: {cron: "0 9 * * *"},
|
|
120
|
+
});
|
|
121
|
+
registerJobsApp(jobsApp);
|
|
122
|
+
|
|
123
|
+
await jobsApp.startWorker();
|
|
124
|
+
await jobsApp.stopWorker();
|
|
125
|
+
|
|
126
|
+
const schedule = await JobSchedule.findExactlyOne({name: "app-tz-daily"});
|
|
127
|
+
assert.equal(schedule.timezone, "America/New_York");
|
|
128
|
+
});
|
|
129
|
+
|
|
130
|
+
it("enqueues a recurring child job with scheduleId when due", async (): Promise<void> => {
|
|
131
|
+
let handlerRuns = 0;
|
|
132
|
+
const jobsApp = new JobsApp({pollIntervalMs: 25});
|
|
133
|
+
jobsApp.define("recurring-child", {
|
|
134
|
+
handler: async () => {
|
|
135
|
+
handlerRuns += 1;
|
|
136
|
+
},
|
|
137
|
+
schedule: {cron: "0 9 * * *", timezone: "UTC"},
|
|
138
|
+
});
|
|
139
|
+
registerJobsApp(jobsApp);
|
|
140
|
+
|
|
141
|
+
await jobsApp.startWorker();
|
|
142
|
+
const schedule = await JobSchedule.findExactlyOne({name: "recurring-child"});
|
|
143
|
+
await JobSchedule.updateOne(
|
|
144
|
+
{_id: schedule._id},
|
|
145
|
+
{$set: {nextRunAt: DateTime.utc().minus({minutes: 1}).toJSDate()}}
|
|
146
|
+
);
|
|
147
|
+
|
|
148
|
+
await waitUntil(async () => handlerRuns >= 1);
|
|
149
|
+
await jobsApp.stopWorker();
|
|
150
|
+
|
|
151
|
+
const childJobs = await Job.find({name: "recurring-child"});
|
|
152
|
+
assert.equal(childJobs.length, 1);
|
|
153
|
+
assert.equal(childJobs[0]?.scheduleId?.toString(), schedule._id.toString());
|
|
154
|
+
assert.equal(handlerRuns, 1);
|
|
155
|
+
|
|
156
|
+
const updatedSchedule = await JobSchedule.findExactlyOne({_id: schedule._id});
|
|
157
|
+
assert.isAbove(updatedSchedule.nextRunAt.getTime(), DateTime.utc().toMillis() - 1_000);
|
|
158
|
+
});
|
|
159
|
+
|
|
160
|
+
it("skips enqueue when a child is still in-flight but advances nextRunAt", async (): Promise<void> => {
|
|
161
|
+
const jobsApp = new JobsApp({pollIntervalMs: 25});
|
|
162
|
+
jobsApp.define("overlap-skip", {
|
|
163
|
+
handler: async () => {},
|
|
164
|
+
schedule: {cron: "0 9 * * *", timezone: "UTC"},
|
|
165
|
+
});
|
|
166
|
+
registerJobsApp(jobsApp);
|
|
167
|
+
|
|
168
|
+
await jobsApp.startWorker();
|
|
169
|
+
const schedule = await JobSchedule.findExactlyOne({name: "overlap-skip"});
|
|
170
|
+
const staleNextRunAt = DateTime.utc().minus({minutes: 1}).toJSDate();
|
|
171
|
+
await JobSchedule.updateOne({_id: schedule._id}, {$set: {nextRunAt: staleNextRunAt}});
|
|
172
|
+
|
|
173
|
+
await getJobsService().enqueue({
|
|
174
|
+
name: "overlap-skip",
|
|
175
|
+
payload: {},
|
|
176
|
+
scheduleId: schedule._id.toString(),
|
|
177
|
+
});
|
|
178
|
+
|
|
179
|
+
await waitUntil(async () => {
|
|
180
|
+
const updated = await JobSchedule.findExactlyOne({_id: schedule._id});
|
|
181
|
+
return updated.nextRunAt.getTime() > staleNextRunAt.getTime();
|
|
182
|
+
});
|
|
183
|
+
await jobsApp.stopWorker();
|
|
184
|
+
|
|
185
|
+
const childJobs = await Job.find({name: "overlap-skip"});
|
|
186
|
+
assert.equal(childJobs.length, 1, "overlap tick must not enqueue a second child");
|
|
187
|
+
});
|
|
188
|
+
|
|
189
|
+
it("stores America/New_York nextRunAt differently from UTC for the same cron", async (): Promise<void> => {
|
|
190
|
+
const utcApp = new JobsApp({pollIntervalMs: 25});
|
|
191
|
+
utcApp.define("tz-compare-utc", {
|
|
192
|
+
handler: async () => {},
|
|
193
|
+
schedule: {cron: "0 9 * * *", timezone: "UTC"},
|
|
194
|
+
});
|
|
195
|
+
registerJobsApp(utcApp);
|
|
196
|
+
await utcApp.startWorker();
|
|
197
|
+
await utcApp.stopWorker();
|
|
198
|
+
|
|
199
|
+
const newYorkApp = new JobsApp({pollIntervalMs: 25});
|
|
200
|
+
newYorkApp.define("tz-compare-ny", {
|
|
201
|
+
handler: async () => {},
|
|
202
|
+
schedule: {cron: "0 9 * * *", timezone: "America/New_York"},
|
|
203
|
+
});
|
|
204
|
+
registerJobsApp(newYorkApp);
|
|
205
|
+
await newYorkApp.startWorker();
|
|
206
|
+
await newYorkApp.stopWorker();
|
|
207
|
+
|
|
208
|
+
const utcSchedule = await JobSchedule.findExactlyOne({name: "tz-compare-utc"});
|
|
209
|
+
const newYorkSchedule = await JobSchedule.findExactlyOne({name: "tz-compare-ny"});
|
|
210
|
+
|
|
211
|
+
assert.notEqual(
|
|
212
|
+
utcSchedule.nextRunAt.toISOString(),
|
|
213
|
+
newYorkSchedule.nextRunAt.toISOString(),
|
|
214
|
+
"same cron in different IANA zones must not share nextRunAt"
|
|
215
|
+
);
|
|
216
|
+
});
|
|
217
|
+
|
|
218
|
+
it("preserves overdue nextRunAt and enabled:false across reconcile when cron/timezone unchanged", async (): Promise<void> => {
|
|
219
|
+
const jobsApp = new JobsApp({pollIntervalMs: 25});
|
|
220
|
+
await startScheduledWorker(jobsApp, "paused-overdue", async () => {}, {
|
|
221
|
+
cron: "0 9 * * *",
|
|
222
|
+
timezone: "UTC",
|
|
223
|
+
});
|
|
224
|
+
|
|
225
|
+
const overdue = DateTime.utc().minus({hours: 2}).toJSDate();
|
|
226
|
+
await JobSchedule.updateOne(
|
|
227
|
+
{name: "paused-overdue"},
|
|
228
|
+
{$set: {enabled: false, nextRunAt: overdue}}
|
|
229
|
+
);
|
|
230
|
+
|
|
231
|
+
await getJobsService().reconcileSchedules();
|
|
232
|
+
|
|
233
|
+
const schedule = await JobSchedule.findExactlyOne({name: "paused-overdue"});
|
|
234
|
+
assert.equal(schedule.enabled, false);
|
|
235
|
+
assert.equal(schedule.nextRunAt.toISOString(), overdue.toISOString());
|
|
236
|
+
|
|
237
|
+
await jobsApp.stopWorker();
|
|
238
|
+
});
|
|
239
|
+
|
|
240
|
+
it("does not enqueue when disabled and catches up overdue rows after re-enable", async (): Promise<void> => {
|
|
241
|
+
let handlerRuns = 0;
|
|
242
|
+
const jobsApp = new JobsApp({pollIntervalMs: 25});
|
|
243
|
+
await startScheduledWorker(
|
|
244
|
+
jobsApp,
|
|
245
|
+
"pause-resume",
|
|
246
|
+
async () => {
|
|
247
|
+
handlerRuns += 1;
|
|
248
|
+
},
|
|
249
|
+
{cron: "0 9 * * *", timezone: "UTC"}
|
|
250
|
+
);
|
|
251
|
+
|
|
252
|
+
const overdue = DateTime.utc().minus({minutes: 5}).toJSDate();
|
|
253
|
+
await JobSchedule.updateOne(
|
|
254
|
+
{name: "pause-resume"},
|
|
255
|
+
{$set: {enabled: false, nextRunAt: overdue}}
|
|
256
|
+
);
|
|
257
|
+
|
|
258
|
+
await waitUntil(async () => (await Job.countDocuments({name: "pause-resume"})) === 0, {
|
|
259
|
+
timeoutMs: 500,
|
|
260
|
+
});
|
|
261
|
+
|
|
262
|
+
await JobSchedule.updateOne({name: "pause-resume"}, {$set: {enabled: true}});
|
|
263
|
+
await waitUntil(async () => handlerRuns >= 1);
|
|
264
|
+
|
|
265
|
+
assert.equal(await Job.countDocuments({name: "pause-resume"}), 1);
|
|
266
|
+
await jobsApp.stopWorker();
|
|
267
|
+
});
|
|
268
|
+
|
|
269
|
+
it("updates nextRunAt but preserves enabled:false when cron changes", async (): Promise<void> => {
|
|
270
|
+
const jobsApp = new JobsApp({pollIntervalMs: 25});
|
|
271
|
+
jobsApp.define("cron-change", {
|
|
272
|
+
handler: async () => {},
|
|
273
|
+
schedule: {cron: "0 9 * * *", timezone: "UTC"},
|
|
274
|
+
});
|
|
275
|
+
registerJobsApp(jobsApp);
|
|
276
|
+
await jobsApp.startWorker();
|
|
277
|
+
|
|
278
|
+
const before = await JobSchedule.findExactlyOne({name: "cron-change"});
|
|
279
|
+
await JobSchedule.updateOne({name: "cron-change"}, {$set: {enabled: false}});
|
|
280
|
+
|
|
281
|
+
jobsApp.define("cron-change", {
|
|
282
|
+
handler: async () => {},
|
|
283
|
+
schedule: {cron: "0 10 * * *", timezone: "UTC"},
|
|
284
|
+
});
|
|
285
|
+
getJobsService().markSchedulesDirty();
|
|
286
|
+
await getJobsService().reconcileSchedulesIfDirty();
|
|
287
|
+
|
|
288
|
+
const after = await JobSchedule.findExactlyOne({name: "cron-change"});
|
|
289
|
+
assert.equal(after.enabled, false);
|
|
290
|
+
assert.notEqual(after.nextRunAt.toISOString(), before.nextRunAt.toISOString());
|
|
291
|
+
assert.equal(after.cron, "0 10 * * *");
|
|
292
|
+
|
|
293
|
+
await jobsApp.stopWorker();
|
|
294
|
+
});
|
|
295
|
+
|
|
296
|
+
it("persists a schedule defined after the worker has started via dirty reconcile", async (): Promise<void> => {
|
|
297
|
+
const jobsApp = new JobsApp({pollIntervalMs: 25});
|
|
298
|
+
jobsApp.define("existing", {
|
|
299
|
+
handler: async () => {},
|
|
300
|
+
});
|
|
301
|
+
registerJobsApp(jobsApp);
|
|
302
|
+
await jobsApp.startWorker();
|
|
303
|
+
|
|
304
|
+
jobsApp.define("late-schedule", {
|
|
305
|
+
handler: async () => {},
|
|
306
|
+
schedule: {cron: "0 9 * * *", timezone: "UTC"},
|
|
307
|
+
});
|
|
308
|
+
|
|
309
|
+
await waitUntil(async () => (await JobSchedule.countDocuments({name: "late-schedule"})) === 1);
|
|
310
|
+
assert.isFalse(getJobsService().isSchedulesDirty());
|
|
311
|
+
|
|
312
|
+
await jobsApp.stopWorker();
|
|
313
|
+
});
|
|
314
|
+
|
|
315
|
+
it("creates exactly one child when two workers race the same due schedule", async (): Promise<void> => {
|
|
316
|
+
let handlerRuns = 0;
|
|
317
|
+
const handler = async (): Promise<void> => {
|
|
318
|
+
handlerRuns += 1;
|
|
319
|
+
};
|
|
320
|
+
|
|
321
|
+
const workerA = new JobsApp({pollIntervalMs: 25});
|
|
322
|
+
const workerB = new JobsApp({pollIntervalMs: 25});
|
|
323
|
+
workerA.define("race-schedule", {handler, schedule: {cron: "0 9 * * *", timezone: "UTC"}});
|
|
324
|
+
workerB.define("race-schedule", {handler, schedule: {cron: "0 9 * * *", timezone: "UTC"}});
|
|
325
|
+
registerJobsApp(workerA);
|
|
326
|
+
|
|
327
|
+
try {
|
|
328
|
+
await Promise.all([workerA.startWorker(), workerB.startWorker()]);
|
|
329
|
+
|
|
330
|
+
const schedule = await JobSchedule.findExactlyOne({name: "race-schedule"});
|
|
331
|
+
await JobSchedule.updateOne(
|
|
332
|
+
{_id: schedule._id},
|
|
333
|
+
{$set: {nextRunAt: DateTime.utc().minus({minutes: 1}).toJSDate()}}
|
|
334
|
+
);
|
|
335
|
+
|
|
336
|
+
await waitUntil(async () => handlerRuns >= 1);
|
|
337
|
+
|
|
338
|
+
assert.equal(await Job.countDocuments({name: "race-schedule"}), 1);
|
|
339
|
+
assert.equal(handlerRuns, 1);
|
|
340
|
+
} finally {
|
|
341
|
+
await Promise.allSettled([workerA.stopWorker(), workerB.stopWorker()]);
|
|
342
|
+
}
|
|
343
|
+
});
|
|
344
|
+
|
|
345
|
+
it("rolls back nextRunAt and retries enqueue so exactly one child is created", async (): Promise<void> => {
|
|
346
|
+
const jobsApp = new JobsApp({pollIntervalMs: 25});
|
|
347
|
+
jobsApp.define("enqueue-retry", {
|
|
348
|
+
handler: async () => {},
|
|
349
|
+
schedule: {cron: "0 9 * * *", timezone: "UTC"},
|
|
350
|
+
});
|
|
351
|
+
registerJobsApp(jobsApp);
|
|
352
|
+
await jobsApp.startWorker();
|
|
353
|
+
|
|
354
|
+
const schedule = await JobSchedule.findExactlyOne({name: "enqueue-retry"});
|
|
355
|
+
const dueRunAt = DateTime.utc().minus({minutes: 1}).toJSDate();
|
|
356
|
+
await JobSchedule.updateOne({_id: schedule._id}, {$set: {nextRunAt: dueRunAt}});
|
|
357
|
+
|
|
358
|
+
const service = getJobsService();
|
|
359
|
+
const originalEnqueue = service.enqueue.bind(service);
|
|
360
|
+
let enqueueAttempts = 0;
|
|
361
|
+
service.enqueue = async (params) => {
|
|
362
|
+
enqueueAttempts += 1;
|
|
363
|
+
if (enqueueAttempts === 1) {
|
|
364
|
+
throw new Error("simulated enqueue failure");
|
|
365
|
+
}
|
|
366
|
+
return originalEnqueue(params);
|
|
367
|
+
};
|
|
368
|
+
|
|
369
|
+
await tickDueSchedules({jobsService: service, now: DateTime.utc().toJSDate()});
|
|
370
|
+
|
|
371
|
+
const rolledBack = await JobSchedule.findExactlyOne({_id: schedule._id});
|
|
372
|
+
assert.equal(rolledBack.nextRunAt.toISOString(), dueRunAt.toISOString());
|
|
373
|
+
assert.equal(await Job.countDocuments({name: "enqueue-retry"}), 0);
|
|
374
|
+
|
|
375
|
+
await tickDueSchedules({jobsService: service, now: DateTime.utc().toJSDate()});
|
|
376
|
+
|
|
377
|
+
assert.equal(enqueueAttempts, 2);
|
|
378
|
+
assert.equal(await Job.countDocuments({name: "enqueue-retry"}), 1);
|
|
379
|
+
assert.equal(
|
|
380
|
+
(await Job.findExactlyOne({name: "enqueue-retry"})).idempotencyKey,
|
|
381
|
+
buildScheduleRunIdempotencyKey(schedule._id.toString(), dueRunAt)
|
|
382
|
+
);
|
|
383
|
+
|
|
384
|
+
await jobsApp.stopWorker();
|
|
385
|
+
});
|
|
386
|
+
|
|
387
|
+
it("isolates tick errors so other due schedules still enqueue", async (): Promise<void> => {
|
|
388
|
+
const jobsApp = new JobsApp({pollIntervalMs: 25});
|
|
389
|
+
jobsApp.define("fail-tick", {
|
|
390
|
+
handler: async () => {},
|
|
391
|
+
schedule: {cron: "0 9 * * *", timezone: "UTC"},
|
|
392
|
+
});
|
|
393
|
+
jobsApp.define("ok-tick", {
|
|
394
|
+
handler: async () => {},
|
|
395
|
+
schedule: {cron: "0 9 * * *", timezone: "UTC"},
|
|
396
|
+
});
|
|
397
|
+
registerJobsApp(jobsApp);
|
|
398
|
+
await jobsApp.startWorker();
|
|
399
|
+
|
|
400
|
+
const due = DateTime.utc().minus({minutes: 1}).toJSDate();
|
|
401
|
+
await JobSchedule.updateMany({}, {$set: {nextRunAt: due}});
|
|
402
|
+
|
|
403
|
+
const service = getJobsService();
|
|
404
|
+
const originalEnqueue = service.enqueue.bind(service);
|
|
405
|
+
service.enqueue = async (params) => {
|
|
406
|
+
if (params.name === "fail-tick") {
|
|
407
|
+
throw new Error("simulated tick enqueue failure");
|
|
408
|
+
}
|
|
409
|
+
return originalEnqueue(params);
|
|
410
|
+
};
|
|
411
|
+
|
|
412
|
+
await tickDueSchedules({jobsService: service, now: DateTime.utc().toJSDate()});
|
|
413
|
+
|
|
414
|
+
assert.equal(await Job.countDocuments({name: "fail-tick"}), 0);
|
|
415
|
+
assert.equal(await Job.countDocuments({name: "ok-tick"}), 1);
|
|
416
|
+
|
|
417
|
+
await jobsApp.stopWorker();
|
|
418
|
+
});
|
|
419
|
+
|
|
420
|
+
it("keeps the worker running after a schedule tick enqueue failure", async (): Promise<void> => {
|
|
421
|
+
let manualRuns = 0;
|
|
422
|
+
const jobsApp = new JobsApp({pollIntervalMs: 25});
|
|
423
|
+
jobsApp.define("fail-tick-worker", {
|
|
424
|
+
handler: async () => {},
|
|
425
|
+
schedule: {cron: "0 9 * * *", timezone: "UTC"},
|
|
426
|
+
});
|
|
427
|
+
jobsApp.define("manual-job", {
|
|
428
|
+
handler: async () => {
|
|
429
|
+
manualRuns += 1;
|
|
430
|
+
},
|
|
431
|
+
});
|
|
432
|
+
registerJobsApp(jobsApp);
|
|
433
|
+
await jobsApp.startWorker();
|
|
434
|
+
|
|
435
|
+
const due = DateTime.utc().minus({minutes: 1}).toJSDate();
|
|
436
|
+
await JobSchedule.updateOne({name: "fail-tick-worker"}, {$set: {nextRunAt: due}});
|
|
437
|
+
|
|
438
|
+
const service = getJobsService();
|
|
439
|
+
const originalEnqueue = service.enqueue.bind(service);
|
|
440
|
+
service.enqueue = async (params) => {
|
|
441
|
+
if (params.name === "fail-tick-worker") {
|
|
442
|
+
throw new Error("simulated tick enqueue failure");
|
|
443
|
+
}
|
|
444
|
+
return originalEnqueue(params);
|
|
445
|
+
};
|
|
446
|
+
|
|
447
|
+
await getJobsService().enqueue({name: "manual-job", payload: {}});
|
|
448
|
+
|
|
449
|
+
await waitUntil(async () => manualRuns >= 1);
|
|
450
|
+
assert.isTrue(jobsApp.isWorkerActive());
|
|
451
|
+
|
|
452
|
+
await jobsApp.stopWorker();
|
|
453
|
+
});
|
|
454
|
+
|
|
455
|
+
it("survives worker restart with overdue nextRunAt intact until caught up", async (): Promise<void> => {
|
|
456
|
+
let handlerRuns = 0;
|
|
457
|
+
const jobsApp = new JobsApp({pollIntervalMs: 25});
|
|
458
|
+
await startScheduledWorker(
|
|
459
|
+
jobsApp,
|
|
460
|
+
"restart-catchup",
|
|
461
|
+
async () => {
|
|
462
|
+
handlerRuns += 1;
|
|
463
|
+
},
|
|
464
|
+
{cron: "0 9 * * *", timezone: "UTC"}
|
|
465
|
+
);
|
|
466
|
+
|
|
467
|
+
await jobsApp.stopWorker();
|
|
468
|
+
|
|
469
|
+
const overdue = DateTime.utc().minus({minutes: 10}).toJSDate();
|
|
470
|
+
await JobSchedule.updateOne({name: "restart-catchup"}, {$set: {nextRunAt: overdue}});
|
|
471
|
+
|
|
472
|
+
await getJobsService().reconcileSchedules();
|
|
473
|
+
const afterReconcile = await JobSchedule.findExactlyOne({name: "restart-catchup"});
|
|
474
|
+
assert.equal(afterReconcile.nextRunAt.toISOString(), overdue.toISOString());
|
|
475
|
+
|
|
476
|
+
await jobsApp.startWorker();
|
|
477
|
+
await waitUntil(async () => handlerRuns >= 1);
|
|
478
|
+
await jobsApp.stopWorker();
|
|
479
|
+
|
|
480
|
+
assert.equal(handlerRuns, 1);
|
|
481
|
+
assert.equal(await Job.countDocuments({name: "restart-catchup"}), 1);
|
|
482
|
+
});
|
|
483
|
+
});
|
|
484
|
+
|
|
485
|
+
describe("jobs schedule reconcile unit", () => {
|
|
486
|
+
beforeEach(async (): Promise<void> => {
|
|
487
|
+
await setupDb();
|
|
488
|
+
await Job.deleteMany({});
|
|
489
|
+
await JobSchedule.deleteMany({});
|
|
490
|
+
});
|
|
491
|
+
|
|
492
|
+
it("initializes enabled and nextRunAt on first reconcile insert", async (): Promise<void> => {
|
|
493
|
+
const service = new JobsService({defaultTimezone: "UTC"});
|
|
494
|
+
service.define("fresh", {
|
|
495
|
+
handler: async () => {},
|
|
496
|
+
schedule: {cron: "0 9 * * *", timezone: "UTC"},
|
|
497
|
+
});
|
|
498
|
+
|
|
499
|
+
await service.reconcileSchedules();
|
|
500
|
+
|
|
501
|
+
const schedule = await JobSchedule.findExactlyOne({name: "fresh"});
|
|
502
|
+
assert.equal(schedule.enabled, true);
|
|
503
|
+
assert.isAbove(schedule.nextRunAt.getTime(), DateTime.utc().toMillis() - 1_000);
|
|
504
|
+
});
|
|
505
|
+
});
|
|
@@ -0,0 +1,165 @@
|
|
|
1
|
+
import {beforeEach, describe, it} from "bun:test";
|
|
2
|
+
import {
|
|
3
|
+
type APIError,
|
|
4
|
+
getCurrentRequestContext,
|
|
5
|
+
isAPIError,
|
|
6
|
+
TerrenoApp,
|
|
7
|
+
type UserModel as UserModelType,
|
|
8
|
+
} from "@terreno/api";
|
|
9
|
+
import {setupDb, UserModel} from "@terreno/api/testing";
|
|
10
|
+
import {assert} from "chai";
|
|
11
|
+
import {DateTime} from "luxon";
|
|
12
|
+
import {JobsApp} from "../jobsApp";
|
|
13
|
+
import {getJobsService} from "../jobsService";
|
|
14
|
+
import {Job} from "../models/job";
|
|
15
|
+
import {MongoJobRunner} from "../runners/mongoRunner";
|
|
16
|
+
|
|
17
|
+
const typedUserModel = UserModel as unknown as UserModelType;
|
|
18
|
+
|
|
19
|
+
const waitUntil = async (
|
|
20
|
+
predicate: () => Promise<boolean>,
|
|
21
|
+
options?: {intervalMs?: number; timeoutMs?: number}
|
|
22
|
+
): Promise<void> => {
|
|
23
|
+
const intervalMs = options?.intervalMs ?? 25;
|
|
24
|
+
const timeoutMs = options?.timeoutMs ?? 5_000;
|
|
25
|
+
const deadline = DateTime.utc().plus({milliseconds: timeoutMs});
|
|
26
|
+
|
|
27
|
+
while (DateTime.utc() < deadline) {
|
|
28
|
+
if (await predicate()) {
|
|
29
|
+
return;
|
|
30
|
+
}
|
|
31
|
+
await Bun.sleep(intervalMs);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
throw new Error("waitUntil timed out");
|
|
35
|
+
};
|
|
36
|
+
|
|
37
|
+
const captureError = async (operation: () => Promise<unknown>): Promise<unknown> => {
|
|
38
|
+
try {
|
|
39
|
+
await operation();
|
|
40
|
+
return undefined;
|
|
41
|
+
} catch (error: unknown) {
|
|
42
|
+
return error;
|
|
43
|
+
}
|
|
44
|
+
};
|
|
45
|
+
|
|
46
|
+
describe("jobs tracer", () => {
|
|
47
|
+
beforeEach(async (): Promise<void> => {
|
|
48
|
+
await setupDb();
|
|
49
|
+
await Job.deleteMany({});
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
it("enqueue then startWorker completes the job inside runWithRequestContext", async (): Promise<void> => {
|
|
53
|
+
let seenPayload: unknown;
|
|
54
|
+
let seenJobId: string | undefined;
|
|
55
|
+
let contextJobId: string | undefined;
|
|
56
|
+
|
|
57
|
+
const jobsApp = new JobsApp({pollIntervalMs: 25});
|
|
58
|
+
jobsApp.define("echo-payload", {
|
|
59
|
+
handler: async (payload, ctx) => {
|
|
60
|
+
seenPayload = payload;
|
|
61
|
+
seenJobId = ctx.jobId;
|
|
62
|
+
contextJobId = getCurrentRequestContext()?.jobId;
|
|
63
|
+
},
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
new TerrenoApp({
|
|
67
|
+
skipListen: true,
|
|
68
|
+
userModel: typedUserModel,
|
|
69
|
+
})
|
|
70
|
+
.register(jobsApp)
|
|
71
|
+
.build();
|
|
72
|
+
|
|
73
|
+
const enqueued = await getJobsService().enqueue({
|
|
74
|
+
name: "echo-payload",
|
|
75
|
+
payload: {value: "hello"},
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
assert.equal(enqueued.status, "pending");
|
|
79
|
+
assert.deepEqual(enqueued.payload, {value: "hello"});
|
|
80
|
+
assert.isAtMost(
|
|
81
|
+
Math.abs(DateTime.fromJSDate(enqueued.runAt).toMillis() - DateTime.utc().toMillis()),
|
|
82
|
+
5_000
|
|
83
|
+
);
|
|
84
|
+
|
|
85
|
+
await jobsApp.startWorker();
|
|
86
|
+
await waitUntil(
|
|
87
|
+
async () => (await Job.findExactlyOne({_id: enqueued._id})).status === "completed"
|
|
88
|
+
);
|
|
89
|
+
await jobsApp.stopWorker();
|
|
90
|
+
|
|
91
|
+
const completed = await Job.findExactlyOne({_id: enqueued._id});
|
|
92
|
+
assert.equal(completed.status, "completed");
|
|
93
|
+
assert.deepEqual(seenPayload, {value: "hello"});
|
|
94
|
+
assert.equal(seenJobId, enqueued._id.toString());
|
|
95
|
+
assert.equal(contextJobId, enqueued._id.toString());
|
|
96
|
+
});
|
|
97
|
+
|
|
98
|
+
it("enqueue with an unknown job name throws APIError 400", async (): Promise<void> => {
|
|
99
|
+
const jobsApp = new JobsApp({pollIntervalMs: 25});
|
|
100
|
+
new TerrenoApp({
|
|
101
|
+
skipListen: true,
|
|
102
|
+
userModel: typedUserModel,
|
|
103
|
+
})
|
|
104
|
+
.register(jobsApp)
|
|
105
|
+
.build();
|
|
106
|
+
|
|
107
|
+
const error = await captureError(() =>
|
|
108
|
+
getJobsService().enqueue({
|
|
109
|
+
name: "missing-handler",
|
|
110
|
+
payload: {},
|
|
111
|
+
})
|
|
112
|
+
);
|
|
113
|
+
|
|
114
|
+
assert.isTrue(isAPIError(error));
|
|
115
|
+
assert.equal((error as APIError).status, 400);
|
|
116
|
+
assert.equal(await Job.countDocuments(), 0);
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
it("fails loud when a pending row has no registered handler at execution time", async (): Promise<void> => {
|
|
120
|
+
const jobsApp = new JobsApp({pollIntervalMs: 25});
|
|
121
|
+
new TerrenoApp({
|
|
122
|
+
skipListen: true,
|
|
123
|
+
userModel: typedUserModel,
|
|
124
|
+
})
|
|
125
|
+
.register(jobsApp)
|
|
126
|
+
.build();
|
|
127
|
+
|
|
128
|
+
await Job.create({
|
|
129
|
+
attemptCount: 0,
|
|
130
|
+
maxAttempts: 5,
|
|
131
|
+
name: "orphan-job",
|
|
132
|
+
payload: {},
|
|
133
|
+
payloadRedacted: false,
|
|
134
|
+
runAt: DateTime.utc().toJSDate(),
|
|
135
|
+
status: "pending",
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
await jobsApp.startWorker();
|
|
139
|
+
await waitUntil(
|
|
140
|
+
async () => (await Job.findExactlyOne({name: "orphan-job"})).status === "failed"
|
|
141
|
+
);
|
|
142
|
+
await jobsApp.stopWorker();
|
|
143
|
+
|
|
144
|
+
const failed = await Job.findExactlyOne({name: "orphan-job"});
|
|
145
|
+
assert.equal(failed.status, "failed");
|
|
146
|
+
assert.match(failed.lastError ?? "", /No handler registered/);
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
it("MongoJobRunner enqueue is a no-op after persistence", async (): Promise<void> => {
|
|
150
|
+
const runner = new MongoJobRunner();
|
|
151
|
+
const job = await Job.create({
|
|
152
|
+
attemptCount: 0,
|
|
153
|
+
maxAttempts: 5,
|
|
154
|
+
name: "noop",
|
|
155
|
+
payload: {},
|
|
156
|
+
payloadRedacted: false,
|
|
157
|
+
runAt: DateTime.utc().toJSDate(),
|
|
158
|
+
status: "pending",
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
await runner.enqueue(job);
|
|
162
|
+
assert.equal(await Job.countDocuments(), 1);
|
|
163
|
+
await Job.deleteMany({name: "noop"});
|
|
164
|
+
});
|
|
165
|
+
});
|