@agendajs/redis-backend 1.0.0-alpha.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +25 -0
- package/README.md +186 -0
- package/dist/RedisBackend.d.ts +73 -0
- package/dist/RedisBackend.js +114 -0
- package/dist/RedisBackend.js.map +1 -0
- package/dist/RedisJobRepository.d.ts +75 -0
- package/dist/RedisJobRepository.js +750 -0
- package/dist/RedisJobRepository.js.map +1 -0
- package/dist/RedisNotificationChannel.d.ts +48 -0
- package/dist/RedisNotificationChannel.js +180 -0
- package/dist/RedisNotificationChannel.js.map +1 -0
- package/dist/index.d.ts +31 -0
- package/dist/index.js +30 -0
- package/dist/index.js.map +1 -0
- package/dist/types.d.ts +49 -0
- package/dist/types.js +2 -0
- package/dist/types.js.map +1 -0
- package/package.json +67 -0
|
@@ -0,0 +1,750 @@
|
|
|
1
|
+
import debug from 'debug';
|
|
2
|
+
import { Redis } from 'ioredis';
|
|
3
|
+
import { randomUUID } from 'crypto';
|
|
4
|
+
import { toJobId, computeJobState } from 'agenda';
|
|
5
|
+
const log = debug('agenda:redis:repository');
|
|
6
|
+
/**
|
|
7
|
+
* Redis implementation of JobRepository
|
|
8
|
+
*
|
|
9
|
+
* Data structure:
|
|
10
|
+
* - `{prefix}job:{id}` - Hash containing job data
|
|
11
|
+
* - `{prefix}jobs:all` - Set of all job IDs
|
|
12
|
+
* - `{prefix}jobs:by_name:{name}` - Set of job IDs by name
|
|
13
|
+
* - `{prefix}jobs:by_next_run_at` - Sorted set of job IDs by nextRunAt timestamp
|
|
14
|
+
* - `{prefix}jobs:single:{name}` - String storing ID of single-type job for a name
|
|
15
|
+
*/
|
|
16
|
+
export class RedisJobRepository {
|
|
17
|
+
config;
|
|
18
|
+
redis;
|
|
19
|
+
ownClient;
|
|
20
|
+
keyPrefix;
|
|
21
|
+
sort;
|
|
22
|
+
constructor(config) {
|
|
23
|
+
this.config = config;
|
|
24
|
+
this.keyPrefix = config.keyPrefix || 'agenda:';
|
|
25
|
+
this.sort = config.sort || { nextRunAt: 'asc', priority: 'desc' };
|
|
26
|
+
// Use existing client or create a new one
|
|
27
|
+
if (config.redis) {
|
|
28
|
+
this.redis = config.redis;
|
|
29
|
+
this.ownClient = false;
|
|
30
|
+
}
|
|
31
|
+
else if (config.connectionString) {
|
|
32
|
+
this.redis = new Redis(config.connectionString);
|
|
33
|
+
this.ownClient = true;
|
|
34
|
+
}
|
|
35
|
+
else if (config.redisOptions) {
|
|
36
|
+
this.redis = new Redis(config.redisOptions);
|
|
37
|
+
this.ownClient = true;
|
|
38
|
+
}
|
|
39
|
+
else {
|
|
40
|
+
throw new Error('RedisBackend requires redis, connectionString, or redisOptions');
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
/**
|
|
44
|
+
* Get the underlying Redis client (for use by notification channel)
|
|
45
|
+
*/
|
|
46
|
+
getRedis() {
|
|
47
|
+
return this.redis;
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Generate Redis key with prefix
|
|
51
|
+
*/
|
|
52
|
+
key(suffix) {
|
|
53
|
+
return `${this.keyPrefix}${suffix}`;
|
|
54
|
+
}
|
|
55
|
+
async connect() {
|
|
56
|
+
log('connecting to Redis');
|
|
57
|
+
// Test connection
|
|
58
|
+
await this.redis.ping();
|
|
59
|
+
log('connection successful');
|
|
60
|
+
}
|
|
61
|
+
async disconnect() {
|
|
62
|
+
log('disconnecting from Redis');
|
|
63
|
+
// Only close the client if we created it
|
|
64
|
+
if (this.ownClient) {
|
|
65
|
+
this.redis.disconnect();
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Convert Redis hash data to JobParameters
|
|
70
|
+
*/
|
|
71
|
+
hashToJob(data) {
|
|
72
|
+
return {
|
|
73
|
+
_id: toJobId(data.id),
|
|
74
|
+
name: data.name,
|
|
75
|
+
priority: parseInt(data.priority, 10),
|
|
76
|
+
nextRunAt: data.nextRunAt && data.nextRunAt !== 'null' ? new Date(data.nextRunAt) : null,
|
|
77
|
+
type: data.type,
|
|
78
|
+
lockedAt: data.lockedAt && data.lockedAt !== 'null' ? new Date(data.lockedAt) : undefined,
|
|
79
|
+
lastFinishedAt: data.lastFinishedAt && data.lastFinishedAt !== 'null'
|
|
80
|
+
? new Date(data.lastFinishedAt)
|
|
81
|
+
: undefined,
|
|
82
|
+
failedAt: data.failedAt && data.failedAt !== 'null' ? new Date(data.failedAt) : undefined,
|
|
83
|
+
failCount: data.failCount && data.failCount !== 'null'
|
|
84
|
+
? parseInt(data.failCount, 10)
|
|
85
|
+
: undefined,
|
|
86
|
+
failReason: data.failReason && data.failReason !== 'null' ? data.failReason : undefined,
|
|
87
|
+
repeatTimezone: data.repeatTimezone && data.repeatTimezone !== 'null'
|
|
88
|
+
? data.repeatTimezone
|
|
89
|
+
: undefined,
|
|
90
|
+
lastRunAt: data.lastRunAt && data.lastRunAt !== 'null' ? new Date(data.lastRunAt) : undefined,
|
|
91
|
+
repeatInterval: data.repeatInterval && data.repeatInterval !== 'null'
|
|
92
|
+
? data.repeatInterval
|
|
93
|
+
: undefined,
|
|
94
|
+
data: JSON.parse(data.data || '{}'),
|
|
95
|
+
repeatAt: data.repeatAt && data.repeatAt !== 'null' ? data.repeatAt : undefined,
|
|
96
|
+
disabled: data.disabled === 'true',
|
|
97
|
+
progress: data.progress && data.progress !== 'null' ? parseFloat(data.progress) : undefined,
|
|
98
|
+
fork: data.fork === 'true',
|
|
99
|
+
lastModifiedBy: data.lastModifiedBy && data.lastModifiedBy !== 'null'
|
|
100
|
+
? data.lastModifiedBy
|
|
101
|
+
: undefined
|
|
102
|
+
};
|
|
103
|
+
}
|
|
104
|
+
/**
|
|
105
|
+
* Convert JobParameters to Redis hash data
|
|
106
|
+
*/
|
|
107
|
+
jobToHash(job, id, lastModifiedBy) {
|
|
108
|
+
const now = new Date().toISOString();
|
|
109
|
+
return {
|
|
110
|
+
id,
|
|
111
|
+
name: job.name,
|
|
112
|
+
priority: String(job.priority ?? 0),
|
|
113
|
+
nextRunAt: job.nextRunAt?.toISOString() || 'null',
|
|
114
|
+
type: job.type || 'normal',
|
|
115
|
+
lockedAt: job.lockedAt?.toISOString() || 'null',
|
|
116
|
+
lastFinishedAt: job.lastFinishedAt?.toISOString() || 'null',
|
|
117
|
+
failedAt: job.failedAt?.toISOString() || 'null',
|
|
118
|
+
failCount: job.failCount !== undefined ? String(job.failCount) : 'null',
|
|
119
|
+
failReason: job.failReason ?? 'null',
|
|
120
|
+
repeatTimezone: job.repeatTimezone ?? 'null',
|
|
121
|
+
lastRunAt: job.lastRunAt?.toISOString() || 'null',
|
|
122
|
+
repeatInterval: job.repeatInterval !== undefined ? String(job.repeatInterval) : 'null',
|
|
123
|
+
data: JSON.stringify(job.data ?? {}),
|
|
124
|
+
repeatAt: job.repeatAt ?? 'null',
|
|
125
|
+
disabled: String(job.disabled ?? false),
|
|
126
|
+
progress: job.progress !== undefined ? String(job.progress) : 'null',
|
|
127
|
+
fork: String(job.fork ?? false),
|
|
128
|
+
lastModifiedBy: lastModifiedBy ?? 'null',
|
|
129
|
+
createdAt: now,
|
|
130
|
+
updatedAt: now
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
/**
|
|
134
|
+
* Get score for sorted set based on nextRunAt and priority
|
|
135
|
+
* Score format: timestamp.priority (e.g., 1234567890.005 for priority 5)
|
|
136
|
+
*/
|
|
137
|
+
getJobScore(nextRunAt, priority) {
|
|
138
|
+
if (!nextRunAt) {
|
|
139
|
+
return Number.MAX_SAFE_INTEGER; // Jobs without nextRunAt go to the end
|
|
140
|
+
}
|
|
141
|
+
// Combine timestamp with priority as decimal part
|
|
142
|
+
// Higher priority = lower decimal = comes first when sorting ascending
|
|
143
|
+
const priorityPart = (100 - Math.min(Math.max(priority, 0), 99)) / 1000;
|
|
144
|
+
return nextRunAt.getTime() + priorityPart;
|
|
145
|
+
}
|
|
146
|
+
async getJobById(id) {
|
|
147
|
+
const data = await this.redis.hgetall(this.key(`job:${id}`));
|
|
148
|
+
if (!data || Object.keys(data).length === 0) {
|
|
149
|
+
return null;
|
|
150
|
+
}
|
|
151
|
+
return this.hashToJob(data);
|
|
152
|
+
}
|
|
153
|
+
async queryJobs(options = {}) {
|
|
154
|
+
const { name, names, state, id, ids, search, data, includeDisabled = true, sort, skip = 0, limit = 0 } = options;
|
|
155
|
+
const now = new Date();
|
|
156
|
+
// Get all job IDs to filter
|
|
157
|
+
let jobIds;
|
|
158
|
+
if (id) {
|
|
159
|
+
jobIds = [id];
|
|
160
|
+
}
|
|
161
|
+
else if (ids && ids.length > 0) {
|
|
162
|
+
jobIds = ids.map((i) => i.toString());
|
|
163
|
+
}
|
|
164
|
+
else if (name) {
|
|
165
|
+
jobIds = await this.redis.smembers(this.key(`jobs:by_name:${name}`));
|
|
166
|
+
}
|
|
167
|
+
else if (names && names.length > 0) {
|
|
168
|
+
const sets = names.map((n) => this.key(`jobs:by_name:${n}`));
|
|
169
|
+
jobIds = await this.redis.sunion(...sets);
|
|
170
|
+
}
|
|
171
|
+
else {
|
|
172
|
+
jobIds = await this.redis.smembers(this.key('jobs:all'));
|
|
173
|
+
}
|
|
174
|
+
// Fetch all jobs
|
|
175
|
+
const jobs = [];
|
|
176
|
+
for (const jobId of jobIds) {
|
|
177
|
+
const jobData = await this.redis.hgetall(this.key(`job:${jobId}`));
|
|
178
|
+
if (!jobData || Object.keys(jobData).length === 0)
|
|
179
|
+
continue;
|
|
180
|
+
const job = this.hashToJob(jobData);
|
|
181
|
+
// Apply filters
|
|
182
|
+
if (!includeDisabled && job.disabled)
|
|
183
|
+
continue;
|
|
184
|
+
if (search && !job.name.toLowerCase().includes(search.toLowerCase()))
|
|
185
|
+
continue;
|
|
186
|
+
if (data !== undefined) {
|
|
187
|
+
const jobDataStr = JSON.stringify(job.data);
|
|
188
|
+
const searchDataStr = JSON.stringify(data);
|
|
189
|
+
if (!jobDataStr.includes(searchDataStr.slice(1, -1)))
|
|
190
|
+
continue;
|
|
191
|
+
}
|
|
192
|
+
const jobState = computeJobState(job, now);
|
|
193
|
+
if (state && jobState !== state)
|
|
194
|
+
continue;
|
|
195
|
+
jobs.push({
|
|
196
|
+
...job,
|
|
197
|
+
_id: job._id,
|
|
198
|
+
state: jobState
|
|
199
|
+
});
|
|
200
|
+
}
|
|
201
|
+
// Sort jobs
|
|
202
|
+
this.sortJobs(jobs, sort);
|
|
203
|
+
// Apply pagination
|
|
204
|
+
const total = jobs.length;
|
|
205
|
+
let result = jobs;
|
|
206
|
+
if (limit > 0) {
|
|
207
|
+
result = jobs.slice(skip, skip + limit);
|
|
208
|
+
}
|
|
209
|
+
else if (skip > 0) {
|
|
210
|
+
result = jobs.slice(skip);
|
|
211
|
+
}
|
|
212
|
+
return { jobs: result, total };
|
|
213
|
+
}
|
|
214
|
+
sortJobs(jobs, sort) {
|
|
215
|
+
jobs.sort((a, b) => {
|
|
216
|
+
if (sort) {
|
|
217
|
+
if (sort.nextRunAt !== undefined) {
|
|
218
|
+
const aTime = a.nextRunAt?.getTime() ?? Number.MAX_SAFE_INTEGER;
|
|
219
|
+
const bTime = b.nextRunAt?.getTime() ?? Number.MAX_SAFE_INTEGER;
|
|
220
|
+
const cmp = sort.nextRunAt === 'asc' ? aTime - bTime : bTime - aTime;
|
|
221
|
+
if (cmp !== 0)
|
|
222
|
+
return cmp;
|
|
223
|
+
}
|
|
224
|
+
if (sort.priority !== undefined) {
|
|
225
|
+
const cmp = sort.priority === 'asc' ? a.priority - b.priority : b.priority - a.priority;
|
|
226
|
+
if (cmp !== 0)
|
|
227
|
+
return cmp;
|
|
228
|
+
}
|
|
229
|
+
if (sort.lastRunAt !== undefined) {
|
|
230
|
+
const aTime = a.lastRunAt?.getTime() ?? 0;
|
|
231
|
+
const bTime = b.lastRunAt?.getTime() ?? 0;
|
|
232
|
+
const cmp = sort.lastRunAt === 'asc' ? aTime - bTime : bTime - aTime;
|
|
233
|
+
if (cmp !== 0)
|
|
234
|
+
return cmp;
|
|
235
|
+
}
|
|
236
|
+
if (sort.lastFinishedAt !== undefined) {
|
|
237
|
+
const aTime = a.lastFinishedAt?.getTime() ?? 0;
|
|
238
|
+
const bTime = b.lastFinishedAt?.getTime() ?? 0;
|
|
239
|
+
const cmp = sort.lastFinishedAt === 'asc' ? aTime - bTime : bTime - aTime;
|
|
240
|
+
if (cmp !== 0)
|
|
241
|
+
return cmp;
|
|
242
|
+
}
|
|
243
|
+
if (sort.name !== undefined) {
|
|
244
|
+
const cmp = a.name.localeCompare(b.name);
|
|
245
|
+
return sort.name === 'asc' ? cmp : -cmp;
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
// Default sort: nextRunAt DESC, lastRunAt DESC
|
|
249
|
+
const aNextRun = a.nextRunAt?.getTime() ?? 0;
|
|
250
|
+
const bNextRun = b.nextRunAt?.getTime() ?? 0;
|
|
251
|
+
if (aNextRun !== bNextRun)
|
|
252
|
+
return bNextRun - aNextRun;
|
|
253
|
+
const aLastRun = a.lastRunAt?.getTime() ?? 0;
|
|
254
|
+
const bLastRun = b.lastRunAt?.getTime() ?? 0;
|
|
255
|
+
return bLastRun - aLastRun;
|
|
256
|
+
});
|
|
257
|
+
}
|
|
258
|
+
async getJobsOverview() {
|
|
259
|
+
const now = new Date();
|
|
260
|
+
const names = await this.getDistinctJobNames();
|
|
261
|
+
const overviews = await Promise.all(names.map(async (name) => {
|
|
262
|
+
const jobIds = await this.redis.smembers(this.key(`jobs:by_name:${name}`));
|
|
263
|
+
const overview = {
|
|
264
|
+
name,
|
|
265
|
+
total: jobIds.length,
|
|
266
|
+
running: 0,
|
|
267
|
+
scheduled: 0,
|
|
268
|
+
queued: 0,
|
|
269
|
+
completed: 0,
|
|
270
|
+
failed: 0,
|
|
271
|
+
repeating: 0
|
|
272
|
+
};
|
|
273
|
+
for (const jobId of jobIds) {
|
|
274
|
+
const jobData = await this.redis.hgetall(this.key(`job:${jobId}`));
|
|
275
|
+
if (!jobData || Object.keys(jobData).length === 0)
|
|
276
|
+
continue;
|
|
277
|
+
const job = this.hashToJob(jobData);
|
|
278
|
+
const state = computeJobState(job, now);
|
|
279
|
+
overview[state]++;
|
|
280
|
+
}
|
|
281
|
+
return overview;
|
|
282
|
+
}));
|
|
283
|
+
return overviews;
|
|
284
|
+
}
|
|
285
|
+
async getDistinctJobNames() {
|
|
286
|
+
const pattern = this.key('jobs:by_name:*');
|
|
287
|
+
const keys = await this.redis.keys(pattern);
|
|
288
|
+
const prefix = this.key('jobs:by_name:');
|
|
289
|
+
return keys.map((k) => k.slice(prefix.length)).sort();
|
|
290
|
+
}
|
|
291
|
+
async getQueueSize() {
|
|
292
|
+
const now = Date.now();
|
|
293
|
+
// Get jobs with nextRunAt < now using the sorted set
|
|
294
|
+
const jobIds = await this.redis.zrangebyscore(this.key('jobs:by_next_run_at'), 0, now);
|
|
295
|
+
return jobIds.length;
|
|
296
|
+
}
|
|
297
|
+
async removeJobs(options) {
|
|
298
|
+
let jobIds = [];
|
|
299
|
+
if (options.id) {
|
|
300
|
+
jobIds = [options.id.toString()];
|
|
301
|
+
}
|
|
302
|
+
else if (options.ids && options.ids.length > 0) {
|
|
303
|
+
jobIds = options.ids.map((id) => id.toString());
|
|
304
|
+
}
|
|
305
|
+
else if (options.name) {
|
|
306
|
+
jobIds = await this.redis.smembers(this.key(`jobs:by_name:${options.name}`));
|
|
307
|
+
}
|
|
308
|
+
else if (options.names && options.names.length > 0) {
|
|
309
|
+
const sets = options.names.map((n) => this.key(`jobs:by_name:${n}`));
|
|
310
|
+
jobIds = await this.redis.sunion(...sets);
|
|
311
|
+
}
|
|
312
|
+
else if (options.notNames && options.notNames.length > 0) {
|
|
313
|
+
const allIds = await this.redis.smembers(this.key('jobs:all'));
|
|
314
|
+
const excludeSets = options.notNames.map((n) => this.key(`jobs:by_name:${n}`));
|
|
315
|
+
const excludeIds = new Set(await this.redis.sunion(...excludeSets));
|
|
316
|
+
jobIds = allIds.filter((id) => !excludeIds.has(id));
|
|
317
|
+
}
|
|
318
|
+
else if (options.data !== undefined) {
|
|
319
|
+
// Need to scan all jobs for data match
|
|
320
|
+
const allIds = await this.redis.smembers(this.key('jobs:all'));
|
|
321
|
+
const searchDataStr = JSON.stringify(options.data);
|
|
322
|
+
for (const jobId of allIds) {
|
|
323
|
+
const jobData = await this.redis.hget(this.key(`job:${jobId}`), 'data');
|
|
324
|
+
if (jobData && jobData.includes(searchDataStr.slice(1, -1))) {
|
|
325
|
+
jobIds.push(jobId);
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
}
|
|
329
|
+
else {
|
|
330
|
+
// No criteria - don't delete anything
|
|
331
|
+
return 0;
|
|
332
|
+
}
|
|
333
|
+
if (jobIds.length === 0) {
|
|
334
|
+
return 0;
|
|
335
|
+
}
|
|
336
|
+
let removed = 0;
|
|
337
|
+
for (const jobId of jobIds) {
|
|
338
|
+
const jobData = await this.redis.hgetall(this.key(`job:${jobId}`));
|
|
339
|
+
if (!jobData || Object.keys(jobData).length === 0)
|
|
340
|
+
continue;
|
|
341
|
+
const job = this.hashToJob(jobData);
|
|
342
|
+
await this.deleteJob(jobId, job.name, job.type);
|
|
343
|
+
removed++;
|
|
344
|
+
}
|
|
345
|
+
return removed;
|
|
346
|
+
}
|
|
347
|
+
async deleteJob(id, name, type) {
|
|
348
|
+
const pipeline = this.redis.pipeline();
|
|
349
|
+
// Delete the job hash
|
|
350
|
+
pipeline.del(this.key(`job:${id}`));
|
|
351
|
+
// Remove from indexes
|
|
352
|
+
pipeline.srem(this.key('jobs:all'), id);
|
|
353
|
+
pipeline.srem(this.key(`jobs:by_name:${name}`), id);
|
|
354
|
+
pipeline.zrem(this.key('jobs:by_next_run_at'), id);
|
|
355
|
+
// Remove single job reference if applicable
|
|
356
|
+
if (type === 'single') {
|
|
357
|
+
pipeline.del(this.key(`jobs:single:${name}`));
|
|
358
|
+
}
|
|
359
|
+
await pipeline.exec();
|
|
360
|
+
}
|
|
361
|
+
async unlockJob(job) {
|
|
362
|
+
if (!job._id)
|
|
363
|
+
return;
|
|
364
|
+
const jobId = job._id.toString();
|
|
365
|
+
// Only unlock jobs which are not currently processed (nextRunAt is not null)
|
|
366
|
+
const nextRunAt = await this.redis.hget(this.key(`job:${jobId}`), 'nextRunAt');
|
|
367
|
+
if (nextRunAt && nextRunAt !== 'null') {
|
|
368
|
+
await this.redis.hset(this.key(`job:${jobId}`), 'lockedAt', 'null');
|
|
369
|
+
}
|
|
370
|
+
}
|
|
371
|
+
async unlockJobs(jobIds) {
|
|
372
|
+
if (jobIds.length === 0)
|
|
373
|
+
return;
|
|
374
|
+
for (const jobId of jobIds) {
|
|
375
|
+
const id = jobId.toString();
|
|
376
|
+
await this.redis.hset(this.key(`job:${id}`), 'lockedAt', 'null');
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
async lockJob(job, options) {
|
|
380
|
+
if (!job._id)
|
|
381
|
+
return undefined;
|
|
382
|
+
const jobId = job._id.toString();
|
|
383
|
+
const now = new Date();
|
|
384
|
+
// Use WATCH/MULTI/EXEC for atomic check-and-set
|
|
385
|
+
await this.redis.watch(this.key(`job:${jobId}`));
|
|
386
|
+
try {
|
|
387
|
+
const jobData = await this.redis.hgetall(this.key(`job:${jobId}`));
|
|
388
|
+
if (!jobData || Object.keys(jobData).length === 0) {
|
|
389
|
+
await this.redis.unwatch();
|
|
390
|
+
return undefined;
|
|
391
|
+
}
|
|
392
|
+
// Check conditions
|
|
393
|
+
if (jobData.lockedAt !== 'null') {
|
|
394
|
+
await this.redis.unwatch();
|
|
395
|
+
return undefined;
|
|
396
|
+
}
|
|
397
|
+
if (jobData.disabled === 'true') {
|
|
398
|
+
await this.redis.unwatch();
|
|
399
|
+
return undefined;
|
|
400
|
+
}
|
|
401
|
+
if (jobData.name !== job.name) {
|
|
402
|
+
await this.redis.unwatch();
|
|
403
|
+
return undefined;
|
|
404
|
+
}
|
|
405
|
+
if (job.nextRunAt) {
|
|
406
|
+
const storedNextRunAt = jobData.nextRunAt !== 'null' ? new Date(jobData.nextRunAt) : null;
|
|
407
|
+
if (!storedNextRunAt || storedNextRunAt.getTime() !== job.nextRunAt.getTime()) {
|
|
408
|
+
await this.redis.unwatch();
|
|
409
|
+
return undefined;
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
// Lock the job atomically
|
|
413
|
+
const result = await this.redis
|
|
414
|
+
.multi()
|
|
415
|
+
.hset(this.key(`job:${jobId}`), 'lockedAt', now.toISOString())
|
|
416
|
+
.hset(this.key(`job:${jobId}`), 'lastModifiedBy', options?.lastModifiedBy ?? 'null')
|
|
417
|
+
.hset(this.key(`job:${jobId}`), 'updatedAt', now.toISOString())
|
|
418
|
+
.exec();
|
|
419
|
+
if (!result) {
|
|
420
|
+
// Transaction was aborted (key was modified)
|
|
421
|
+
return undefined;
|
|
422
|
+
}
|
|
423
|
+
// Return the updated job
|
|
424
|
+
const updatedData = await this.redis.hgetall(this.key(`job:${jobId}`));
|
|
425
|
+
return this.hashToJob(updatedData);
|
|
426
|
+
}
|
|
427
|
+
catch {
|
|
428
|
+
await this.redis.unwatch();
|
|
429
|
+
return undefined;
|
|
430
|
+
}
|
|
431
|
+
}
|
|
432
|
+
/**
|
|
433
|
+
* Lua script for atomic find-and-lock operation.
|
|
434
|
+
* This ensures that only one worker can lock a job, even under concurrent access.
|
|
435
|
+
*
|
|
436
|
+
* Arguments:
|
|
437
|
+
* KEYS[1] = jobs:by_name:{jobName} set
|
|
438
|
+
* KEYS[2] = jobs:by_next_run_at sorted set
|
|
439
|
+
* KEYS[3] = job:{id} hash prefix (without the id)
|
|
440
|
+
* ARGV[1] = nextScanAt timestamp (ms)
|
|
441
|
+
* ARGV[2] = lockDeadline timestamp (ms)
|
|
442
|
+
* ARGV[3] = lockTime ISO string
|
|
443
|
+
* ARGV[4] = lastModifiedBy
|
|
444
|
+
* ARGV[5] = sort direction ('asc' or 'desc')
|
|
445
|
+
*
|
|
446
|
+
* Returns: job ID if locked, nil if no job available
|
|
447
|
+
*/
|
|
448
|
+
static FIND_AND_LOCK_SCRIPT = `
|
|
449
|
+
local jobIds = redis.call('SMEMBERS', KEYS[1])
|
|
450
|
+
if #jobIds == 0 then
|
|
451
|
+
return nil
|
|
452
|
+
end
|
|
453
|
+
|
|
454
|
+
local nextScanAt = tonumber(ARGV[1])
|
|
455
|
+
local lockDeadline = tonumber(ARGV[2])
|
|
456
|
+
local lockTime = ARGV[3]
|
|
457
|
+
local lastModifiedBy = ARGV[4]
|
|
458
|
+
local sortDir = ARGV[5]
|
|
459
|
+
local keyPrefix = KEYS[3]
|
|
460
|
+
|
|
461
|
+
-- Get scores and sort candidates
|
|
462
|
+
local candidates = {}
|
|
463
|
+
for _, jobId in ipairs(jobIds) do
|
|
464
|
+
local score = redis.call('ZSCORE', KEYS[2], jobId)
|
|
465
|
+
if score then
|
|
466
|
+
table.insert(candidates, {id = jobId, score = tonumber(score)})
|
|
467
|
+
end
|
|
468
|
+
end
|
|
469
|
+
|
|
470
|
+
-- Sort by score
|
|
471
|
+
if sortDir == 'asc' then
|
|
472
|
+
table.sort(candidates, function(a, b) return a.score < b.score end)
|
|
473
|
+
else
|
|
474
|
+
table.sort(candidates, function(a, b) return a.score > b.score end)
|
|
475
|
+
end
|
|
476
|
+
|
|
477
|
+
-- Try to lock each candidate
|
|
478
|
+
for _, candidate in ipairs(candidates) do
|
|
479
|
+
local jobKey = keyPrefix .. candidate.id
|
|
480
|
+
local jobData = redis.call('HGETALL', jobKey)
|
|
481
|
+
|
|
482
|
+
if #jobData > 0 then
|
|
483
|
+
-- Convert array to table
|
|
484
|
+
local data = {}
|
|
485
|
+
for i = 1, #jobData, 2 do
|
|
486
|
+
data[jobData[i]] = jobData[i + 1]
|
|
487
|
+
end
|
|
488
|
+
|
|
489
|
+
-- Check if disabled
|
|
490
|
+
if data.disabled ~= 'true' then
|
|
491
|
+
local lockedAt = data.lockedAt
|
|
492
|
+
local nextRunAt = data.nextRunAt
|
|
493
|
+
local lockedAtMs = data.lockedAtMs
|
|
494
|
+
|
|
495
|
+
-- Check if job is ready to run
|
|
496
|
+
local isUnlockedAndReady = false
|
|
497
|
+
local isStale = false
|
|
498
|
+
|
|
499
|
+
if lockedAt == 'null' or lockedAt == nil then
|
|
500
|
+
-- Job is unlocked
|
|
501
|
+
if nextRunAt and nextRunAt ~= 'null' then
|
|
502
|
+
-- We compare scores since they're based on nextRunAt
|
|
503
|
+
if candidate.score <= nextScanAt then
|
|
504
|
+
isUnlockedAndReady = true
|
|
505
|
+
end
|
|
506
|
+
end
|
|
507
|
+
else
|
|
508
|
+
-- Job is locked - check if stale using lockedAtMs field
|
|
509
|
+
if lockedAtMs and lockedAtMs ~= 'null' then
|
|
510
|
+
local lockedAtTime = tonumber(lockedAtMs)
|
|
511
|
+
if lockedAtTime and lockedAtTime <= lockDeadline then
|
|
512
|
+
isStale = true
|
|
513
|
+
end
|
|
514
|
+
end
|
|
515
|
+
end
|
|
516
|
+
|
|
517
|
+
if isUnlockedAndReady or isStale then
|
|
518
|
+
-- Lock the job atomically, also store lockedAtMs for efficient stale checks
|
|
519
|
+
local lockTimeMs = redis.call('TIME')
|
|
520
|
+
local lockTimestamp = (lockTimeMs[1] * 1000) + math.floor(lockTimeMs[2] / 1000)
|
|
521
|
+
redis.call('HSET', jobKey, 'lockedAt', lockTime, 'lockedAtMs', tostring(lockTimestamp), 'lastModifiedBy', lastModifiedBy, 'updatedAt', lockTime)
|
|
522
|
+
return candidate.id
|
|
523
|
+
end
|
|
524
|
+
end
|
|
525
|
+
end
|
|
526
|
+
end
|
|
527
|
+
|
|
528
|
+
return nil
|
|
529
|
+
`;
|
|
530
|
+
async getNextJobToRun(jobName, nextScanAt, lockDeadline, now, options) {
|
|
531
|
+
const lockTime = now ?? new Date();
|
|
532
|
+
// Use Lua script for atomic find-and-lock
|
|
533
|
+
const result = await this.redis.eval(RedisJobRepository.FIND_AND_LOCK_SCRIPT, 3, // number of keys
|
|
534
|
+
this.key(`jobs:by_name:${jobName}`), this.key('jobs:by_next_run_at'), this.key('job:'), nextScanAt.getTime().toString(), lockDeadline.getTime().toString(), lockTime.toISOString(), options?.lastModifiedBy ?? 'null', this.sort.nextRunAt || 'asc');
|
|
535
|
+
if (!result) {
|
|
536
|
+
return undefined;
|
|
537
|
+
}
|
|
538
|
+
// Fetch the locked job
|
|
539
|
+
const jobData = await this.redis.hgetall(this.key(`job:${result}`));
|
|
540
|
+
if (!jobData || Object.keys(jobData).length === 0) {
|
|
541
|
+
return undefined;
|
|
542
|
+
}
|
|
543
|
+
return this.hashToJob(jobData);
|
|
544
|
+
}
|
|
545
|
+
async saveJobState(job, options) {
|
|
546
|
+
if (!job._id) {
|
|
547
|
+
throw new Error('Cannot save job state without job ID');
|
|
548
|
+
}
|
|
549
|
+
const jobId = job._id.toString();
|
|
550
|
+
const now = new Date().toISOString();
|
|
551
|
+
// Check if job exists
|
|
552
|
+
const exists = await this.redis.exists(this.key(`job:${jobId}`));
|
|
553
|
+
if (!exists) {
|
|
554
|
+
throw new Error(`job ${job._id} (name: ${job.name}) cannot be updated in the database, maybe it does not exist anymore?`);
|
|
555
|
+
}
|
|
556
|
+
// Update state fields
|
|
557
|
+
const updates = {
|
|
558
|
+
lockedAt: job.lockedAt?.toISOString() || 'null',
|
|
559
|
+
nextRunAt: job.nextRunAt?.toISOString() || 'null',
|
|
560
|
+
lastRunAt: job.lastRunAt?.toISOString() || 'null',
|
|
561
|
+
progress: job.progress !== undefined ? String(job.progress) : 'null',
|
|
562
|
+
failReason: job.failReason ?? 'null',
|
|
563
|
+
failCount: job.failCount !== undefined ? String(job.failCount) : 'null',
|
|
564
|
+
failedAt: job.failedAt?.toISOString() || 'null',
|
|
565
|
+
lastFinishedAt: job.lastFinishedAt?.toISOString() || 'null',
|
|
566
|
+
fork: String(job.fork ?? false),
|
|
567
|
+
lastModifiedBy: options?.lastModifiedBy ?? 'null',
|
|
568
|
+
updatedAt: now
|
|
569
|
+
};
|
|
570
|
+
await this.redis.hset(this.key(`job:${jobId}`), updates);
|
|
571
|
+
// Update the sorted set score for nextRunAt
|
|
572
|
+
if (job.nextRunAt) {
|
|
573
|
+
const score = this.getJobScore(job.nextRunAt, job.priority);
|
|
574
|
+
await this.redis.zadd(this.key('jobs:by_next_run_at'), score, jobId);
|
|
575
|
+
}
|
|
576
|
+
else {
|
|
577
|
+
await this.redis.zadd(this.key('jobs:by_next_run_at'), Number.MAX_SAFE_INTEGER, jobId);
|
|
578
|
+
}
|
|
579
|
+
}
|
|
580
|
+
async saveJob(job, options) {
|
|
581
|
+
log('attempting to save a job');
|
|
582
|
+
const { _id, unique, uniqueOpts, ...props } = job;
|
|
583
|
+
// If the job already has an ID, update it
|
|
584
|
+
if (_id) {
|
|
585
|
+
log('job already has _id, updating');
|
|
586
|
+
const jobId = _id.toString();
|
|
587
|
+
const existingData = await this.redis.hgetall(this.key(`job:${jobId}`));
|
|
588
|
+
if (!existingData || Object.keys(existingData).length === 0) {
|
|
589
|
+
log('job %s was not found for update, returning original data', _id);
|
|
590
|
+
return job;
|
|
591
|
+
}
|
|
592
|
+
// Verify name matches
|
|
593
|
+
if (existingData.name !== props.name) {
|
|
594
|
+
log('job %s name mismatch, returning original data', _id);
|
|
595
|
+
return job;
|
|
596
|
+
}
|
|
597
|
+
// Update the job - only update scheduling/config fields, not execution state fields
|
|
598
|
+
// Execution state (lockedAt, lastRunAt, lastFinishedAt, failedAt, failCount, failReason, progress)
|
|
599
|
+
// should only be updated via saveJobState
|
|
600
|
+
const now = new Date().toISOString();
|
|
601
|
+
const updates = {
|
|
602
|
+
priority: String(props.priority ?? 0),
|
|
603
|
+
nextRunAt: props.nextRunAt?.toISOString() || 'null',
|
|
604
|
+
type: props.type || 'normal',
|
|
605
|
+
repeatTimezone: props.repeatTimezone ?? 'null',
|
|
606
|
+
repeatInterval: props.repeatInterval !== undefined ? String(props.repeatInterval) : 'null',
|
|
607
|
+
data: JSON.stringify(props.data ?? {}),
|
|
608
|
+
repeatAt: props.repeatAt ?? 'null',
|
|
609
|
+
disabled: String(props.disabled ?? false),
|
|
610
|
+
fork: String(props.fork ?? false),
|
|
611
|
+
lastModifiedBy: options?.lastModifiedBy ?? 'null',
|
|
612
|
+
updatedAt: now
|
|
613
|
+
};
|
|
614
|
+
// Only update fail-related fields if they were explicitly set (for job.fail().save() pattern)
|
|
615
|
+
if (props.failReason !== undefined) {
|
|
616
|
+
updates.failReason = props.failReason ?? 'null';
|
|
617
|
+
}
|
|
618
|
+
if (props.failedAt !== undefined) {
|
|
619
|
+
updates.failedAt = props.failedAt?.toISOString() || 'null';
|
|
620
|
+
}
|
|
621
|
+
if (props.failCount !== undefined) {
|
|
622
|
+
updates.failCount = String(props.failCount);
|
|
623
|
+
}
|
|
624
|
+
await this.redis.hset(this.key(`job:${jobId}`), updates);
|
|
625
|
+
// Update sorted set
|
|
626
|
+
const score = this.getJobScore(props.nextRunAt ?? null, props.priority ?? 0);
|
|
627
|
+
await this.redis.zadd(this.key('jobs:by_next_run_at'), score, jobId);
|
|
628
|
+
const updatedData = await this.redis.hgetall(this.key(`job:${jobId}`));
|
|
629
|
+
return this.hashToJob(updatedData);
|
|
630
|
+
}
|
|
631
|
+
// Handle 'single' type jobs - upsert by name
|
|
632
|
+
if (props.type === 'single') {
|
|
633
|
+
log('job with type of "single" found');
|
|
634
|
+
const existingId = await this.redis.get(this.key(`jobs:single:${props.name}`));
|
|
635
|
+
if (existingId) {
|
|
636
|
+
const existingData = await this.redis.hgetall(this.key(`job:${existingId}`));
|
|
637
|
+
if (existingData && Object.keys(existingData).length > 0) {
|
|
638
|
+
// Update existing single job
|
|
639
|
+
const now = new Date();
|
|
640
|
+
const shouldProtectNextRunAt = props.nextRunAt && props.nextRunAt <= now;
|
|
641
|
+
const updates = {
|
|
642
|
+
priority: String(props.priority ?? 0),
|
|
643
|
+
repeatTimezone: props.repeatTimezone ?? 'null',
|
|
644
|
+
repeatInterval: props.repeatInterval !== undefined ? String(props.repeatInterval) : 'null',
|
|
645
|
+
data: JSON.stringify(props.data ?? {}),
|
|
646
|
+
repeatAt: props.repeatAt ?? 'null',
|
|
647
|
+
disabled: String(props.disabled ?? false),
|
|
648
|
+
lastModifiedBy: options?.lastModifiedBy ?? 'null',
|
|
649
|
+
updatedAt: now.toISOString()
|
|
650
|
+
};
|
|
651
|
+
// Only update nextRunAt if not protecting it
|
|
652
|
+
if (!shouldProtectNextRunAt || existingData.nextRunAt === 'null') {
|
|
653
|
+
updates.nextRunAt = props.nextRunAt?.toISOString() || 'null';
|
|
654
|
+
}
|
|
655
|
+
await this.redis.hset(this.key(`job:${existingId}`), updates);
|
|
656
|
+
// Update sorted set
|
|
657
|
+
const nextRunAt = updates.nextRunAt !== undefined
|
|
658
|
+
? updates.nextRunAt
|
|
659
|
+
: existingData.nextRunAt;
|
|
660
|
+
const nextRunAtDate = nextRunAt !== 'null' ? new Date(nextRunAt) : null;
|
|
661
|
+
const score = this.getJobScore(nextRunAtDate, props.priority ?? 0);
|
|
662
|
+
await this.redis.zadd(this.key('jobs:by_next_run_at'), score, existingId);
|
|
663
|
+
const updatedData = await this.redis.hgetall(this.key(`job:${existingId}`));
|
|
664
|
+
return this.hashToJob(updatedData);
|
|
665
|
+
}
|
|
666
|
+
}
|
|
667
|
+
// Create new single job
|
|
668
|
+
const newId = randomUUID();
|
|
669
|
+
const hashData = this.jobToHash(props, newId, options?.lastModifiedBy);
|
|
670
|
+
await this.redis.hset(this.key(`job:${newId}`), hashData);
|
|
671
|
+
await this.redis.sadd(this.key('jobs:all'), newId);
|
|
672
|
+
await this.redis.sadd(this.key(`jobs:by_name:${props.name}`), newId);
|
|
673
|
+
await this.redis.set(this.key(`jobs:single:${props.name}`), newId);
|
|
674
|
+
const score = this.getJobScore(props.nextRunAt ?? null, props.priority ?? 0);
|
|
675
|
+
await this.redis.zadd(this.key('jobs:by_next_run_at'), score, newId);
|
|
676
|
+
const savedData = await this.redis.hgetall(this.key(`job:${newId}`));
|
|
677
|
+
return this.hashToJob(savedData);
|
|
678
|
+
}
|
|
679
|
+
// Handle unique constraint
|
|
680
|
+
if (unique) {
|
|
681
|
+
log('calling upsert with unique constraint');
|
|
682
|
+
// Build query to find existing job
|
|
683
|
+
const allIds = await this.redis.smembers(this.key(`jobs:by_name:${props.name}`));
|
|
684
|
+
for (const jobId of allIds) {
|
|
685
|
+
const jobData = await this.redis.hgetall(this.key(`job:${jobId}`));
|
|
686
|
+
if (!jobData || Object.keys(jobData).length === 0)
|
|
687
|
+
continue;
|
|
688
|
+
let matches = true;
|
|
689
|
+
for (const [key, value] of Object.entries(unique)) {
|
|
690
|
+
if (key.startsWith('data.')) {
|
|
691
|
+
const dataPath = key.substring(5);
|
|
692
|
+
const data = JSON.parse(jobData.data || '{}');
|
|
693
|
+
if (data[dataPath] !== value) {
|
|
694
|
+
matches = false;
|
|
695
|
+
break;
|
|
696
|
+
}
|
|
697
|
+
}
|
|
698
|
+
else {
|
|
699
|
+
const fieldMap = {
|
|
700
|
+
nextRunAt: 'nextRunAt',
|
|
701
|
+
priority: 'priority',
|
|
702
|
+
type: 'type'
|
|
703
|
+
};
|
|
704
|
+
const field = fieldMap[key] || key;
|
|
705
|
+
if (jobData[field] !== String(value)) {
|
|
706
|
+
matches = false;
|
|
707
|
+
break;
|
|
708
|
+
}
|
|
709
|
+
}
|
|
710
|
+
}
|
|
711
|
+
if (matches) {
|
|
712
|
+
if (uniqueOpts?.insertOnly) {
|
|
713
|
+
return this.hashToJob(jobData);
|
|
714
|
+
}
|
|
715
|
+
// Update existing job
|
|
716
|
+
const now = new Date().toISOString();
|
|
717
|
+
const updates = {
|
|
718
|
+
priority: String(props.priority ?? 0),
|
|
719
|
+
nextRunAt: props.nextRunAt?.toISOString() || 'null',
|
|
720
|
+
type: props.type || 'normal',
|
|
721
|
+
repeatTimezone: props.repeatTimezone ?? 'null',
|
|
722
|
+
repeatInterval: props.repeatInterval !== undefined ? String(props.repeatInterval) : 'null',
|
|
723
|
+
data: JSON.stringify(props.data ?? {}),
|
|
724
|
+
repeatAt: props.repeatAt ?? 'null',
|
|
725
|
+
disabled: String(props.disabled ?? false),
|
|
726
|
+
lastModifiedBy: options?.lastModifiedBy ?? 'null',
|
|
727
|
+
updatedAt: now
|
|
728
|
+
};
|
|
729
|
+
await this.redis.hset(this.key(`job:${jobId}`), updates);
|
|
730
|
+
const score = this.getJobScore(props.nextRunAt ?? null, props.priority ?? 0);
|
|
731
|
+
await this.redis.zadd(this.key('jobs:by_next_run_at'), score, jobId);
|
|
732
|
+
const updatedData = await this.redis.hgetall(this.key(`job:${jobId}`));
|
|
733
|
+
return this.hashToJob(updatedData);
|
|
734
|
+
}
|
|
735
|
+
}
|
|
736
|
+
}
|
|
737
|
+
// Insert new job
|
|
738
|
+
log('inserting new job');
|
|
739
|
+
const newId = randomUUID();
|
|
740
|
+
const hashData = this.jobToHash(props, newId, options?.lastModifiedBy);
|
|
741
|
+
await this.redis.hset(this.key(`job:${newId}`), hashData);
|
|
742
|
+
await this.redis.sadd(this.key('jobs:all'), newId);
|
|
743
|
+
await this.redis.sadd(this.key(`jobs:by_name:${props.name}`), newId);
|
|
744
|
+
const score = this.getJobScore(props.nextRunAt ?? null, props.priority ?? 0);
|
|
745
|
+
await this.redis.zadd(this.key('jobs:by_next_run_at'), score, newId);
|
|
746
|
+
const savedData = await this.redis.hgetall(this.key(`job:${newId}`));
|
|
747
|
+
return this.hashToJob(savedData);
|
|
748
|
+
}
|
|
749
|
+
}
|
|
750
|
+
//# sourceMappingURL=RedisJobRepository.js.map
|