bullmq 5.41.5 → 5.41.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/dist/cjs/classes/job-scheduler.js +40 -15
  2. package/dist/cjs/classes/job-scheduler.js.map +1 -1
  3. package/dist/cjs/classes/job.js +9 -12
  4. package/dist/cjs/classes/job.js.map +1 -1
  5. package/dist/cjs/classes/scripts.js +11 -1
  6. package/dist/cjs/classes/scripts.js.map +1 -1
  7. package/dist/cjs/commands/addDelayedJob-6.lua +5 -2
  8. package/dist/cjs/commands/addJobScheduler-10.lua +130 -0
  9. package/dist/cjs/commands/cleanJobsInSet-3.lua +7 -5
  10. package/dist/cjs/commands/includes/addDelayedJob.lua +2 -7
  11. package/dist/cjs/commands/includes/cleanSet.lua +1 -1
  12. package/dist/cjs/commands/includes/getOrSetMaxEvents.lua +6 -6
  13. package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +47 -2
  14. package/dist/cjs/commands/moveStalledJobsToWait-9.lua +1 -1
  15. package/dist/cjs/commands/moveToFinished-14.lua +1 -1
  16. package/dist/cjs/scripts/addDelayedJob-6.js +41 -43
  17. package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
  18. package/dist/cjs/scripts/{addJobScheduler-6.js → addJobScheduler-10.js} +208 -42
  19. package/dist/cjs/scripts/addJobScheduler-10.js.map +1 -0
  20. package/dist/cjs/scripts/addParentJob-4.js +6 -6
  21. package/dist/cjs/scripts/addPrioritizedJob-8.js +6 -6
  22. package/dist/cjs/scripts/addStandardJob-8.js +6 -6
  23. package/dist/cjs/scripts/changeDelay-4.js +6 -6
  24. package/dist/cjs/scripts/cleanJobsInSet-3.js +8 -6
  25. package/dist/cjs/scripts/cleanJobsInSet-3.js.map +1 -1
  26. package/dist/cjs/scripts/index.js +1 -1
  27. package/dist/cjs/scripts/index.js.map +1 -1
  28. package/dist/cjs/scripts/moveJobFromActiveToWait-9.js +6 -6
  29. package/dist/cjs/scripts/moveJobsToWait-8.js +6 -6
  30. package/dist/cjs/scripts/moveStalledJobsToWait-9.js +49 -9
  31. package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
  32. package/dist/cjs/scripts/moveToDelayed-8.js +6 -6
  33. package/dist/cjs/scripts/moveToFinished-14.js +49 -9
  34. package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
  35. package/dist/cjs/scripts/removeJob-3.js +6 -6
  36. package/dist/cjs/scripts/reprocessJob-8.js +6 -6
  37. package/dist/cjs/scripts/retryJob-11.js +6 -6
  38. package/dist/cjs/scripts/updateJobScheduler-7.js +8 -41
  39. package/dist/cjs/scripts/updateJobScheduler-7.js.map +1 -1
  40. package/dist/cjs/scripts/updateProgress-3.js +6 -6
  41. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  42. package/dist/cjs/version.js +1 -1
  43. package/dist/esm/classes/job-scheduler.d.ts +1 -0
  44. package/dist/esm/classes/job-scheduler.js +40 -15
  45. package/dist/esm/classes/job-scheduler.js.map +1 -1
  46. package/dist/esm/classes/job.js +9 -12
  47. package/dist/esm/classes/job.js.map +1 -1
  48. package/dist/esm/classes/scripts.d.ts +2 -2
  49. package/dist/esm/classes/scripts.js +11 -1
  50. package/dist/esm/classes/scripts.js.map +1 -1
  51. package/dist/esm/commands/addDelayedJob-6.lua +5 -2
  52. package/dist/esm/commands/addJobScheduler-10.lua +130 -0
  53. package/dist/esm/commands/cleanJobsInSet-3.lua +7 -5
  54. package/dist/esm/commands/includes/addDelayedJob.lua +2 -7
  55. package/dist/esm/commands/includes/cleanSet.lua +1 -1
  56. package/dist/esm/commands/includes/getOrSetMaxEvents.lua +6 -6
  57. package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +47 -2
  58. package/dist/esm/commands/moveStalledJobsToWait-9.lua +1 -1
  59. package/dist/esm/commands/moveToFinished-14.lua +1 -1
  60. package/dist/esm/scripts/addDelayedJob-6.js +41 -43
  61. package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
  62. package/dist/esm/scripts/{addJobScheduler-6.js → addJobScheduler-10.js} +208 -42
  63. package/dist/esm/scripts/addJobScheduler-10.js.map +1 -0
  64. package/dist/esm/scripts/addParentJob-4.js +6 -6
  65. package/dist/esm/scripts/addPrioritizedJob-8.js +6 -6
  66. package/dist/esm/scripts/addStandardJob-8.js +6 -6
  67. package/dist/esm/scripts/changeDelay-4.js +6 -6
  68. package/dist/esm/scripts/cleanJobsInSet-3.js +8 -6
  69. package/dist/esm/scripts/cleanJobsInSet-3.js.map +1 -1
  70. package/dist/esm/scripts/index.d.ts +1 -1
  71. package/dist/esm/scripts/index.js +1 -1
  72. package/dist/esm/scripts/index.js.map +1 -1
  73. package/dist/esm/scripts/moveJobFromActiveToWait-9.js +6 -6
  74. package/dist/esm/scripts/moveJobsToWait-8.js +6 -6
  75. package/dist/esm/scripts/moveStalledJobsToWait-9.js +49 -9
  76. package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
  77. package/dist/esm/scripts/moveToDelayed-8.js +6 -6
  78. package/dist/esm/scripts/moveToFinished-14.js +49 -9
  79. package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
  80. package/dist/esm/scripts/removeJob-3.js +6 -6
  81. package/dist/esm/scripts/reprocessJob-8.js +6 -6
  82. package/dist/esm/scripts/retryJob-11.js +6 -6
  83. package/dist/esm/scripts/updateJobScheduler-7.js +8 -41
  84. package/dist/esm/scripts/updateJobScheduler-7.js.map +1 -1
  85. package/dist/esm/scripts/updateProgress-3.js +6 -6
  86. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  87. package/dist/esm/version.d.ts +1 -1
  88. package/dist/esm/version.js +1 -1
  89. package/package.json +1 -1
  90. package/dist/cjs/commands/addJobScheduler-6.lua +0 -72
  91. package/dist/cjs/scripts/addJobScheduler-6.js.map +0 -1
  92. package/dist/esm/commands/addJobScheduler-6.lua +0 -72
  93. package/dist/esm/scripts/addJobScheduler-6.js.map +0 -1
  94. /package/dist/esm/scripts/{addJobScheduler-6.d.ts → addJobScheduler-10.d.ts} +0 -0
@@ -4,12 +4,16 @@ exports.addJobScheduler = void 0;
4
4
  const content = `--[[
5
5
  Adds a job scheduler, i.e. a job factory that creates jobs based on a given schedule (repeat options).
6
6
  Input:
7
- KEYS[1] 'repeat' key
8
- KEYS[2] 'delayed' key
9
- KEYS[3] 'wait' key
10
- KEYS[4] 'paused' key
11
- KEYS[5] 'meta' key
12
- KEYS[6] 'prioritized' key
7
+ KEYS[1] 'repeat' key
8
+ KEYS[2] 'delayed' key
9
+ KEYS[3] 'wait' key
10
+ KEYS[4] 'paused' key
11
+ KEYS[5] 'meta' key
12
+ KEYS[6] 'prioritized' key
13
+ KEYS[7] 'marker' key
14
+ KEYS[8] 'id' key
15
+ KEYS[9] 'events' key
16
+ KEYS[10] 'pc' priority counter
13
17
  ARGV[1] next milliseconds
14
18
  ARGV[2] msgpacked options
15
19
  [1] name
@@ -20,18 +24,142 @@ const content = `--[[
20
24
  ARGV[3] jobs scheduler id
21
25
  ARGV[4] Json stringified template data
22
26
  ARGV[5] mspacked template opts
23
- ARGV[6] prefix key
27
+ ARGV[6] msgpacked delayed opts
28
+ ARGV[7] timestamp
29
+ ARGV[8] prefix key
30
+ ARGV[9] producer key
24
31
  Output:
25
32
  repeatableKey - OK
26
- ]] local rcall = redis.call
33
+ ]]
34
+ local rcall = redis.call
27
35
  local repeatKey = KEYS[1]
28
36
  local delayedKey = KEYS[2]
29
37
  local prioritizedKey = KEYS[6]
30
38
  local nextMillis = ARGV[1]
31
39
  local jobSchedulerId = ARGV[3]
32
40
  local templateOpts = cmsgpack.unpack(ARGV[5])
33
- local prefixKey = ARGV[6]
41
+ local prefixKey = ARGV[8]
42
+ -- Includes
43
+ --[[
44
+ Adds a delayed job to the queue by doing the following:
45
+ - Creates a new job key with the job data.
46
+ - adds to delayed zset.
47
+ - Emits a global event 'delayed' if the job is delayed.
48
+ ]]
49
+ -- Includes
50
+ --[[
51
+ Add delay marker if needed.
52
+ ]]
53
+ -- Includes
54
+ --[[
55
+ Function to return the next delayed job timestamp.
56
+ ]]
57
+ local function getNextDelayedTimestamp(delayedKey)
58
+ local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
59
+ if #result then
60
+ local nextTimestamp = tonumber(result[2])
61
+ if nextTimestamp ~= nil then
62
+ return nextTimestamp / 0x1000
63
+ end
64
+ end
65
+ end
66
+ local function addDelayMarkerIfNeeded(markerKey, delayedKey)
67
+ local nextTimestamp = getNextDelayedTimestamp(delayedKey)
68
+ if nextTimestamp ~= nil then
69
+ -- Replace the score of the marker with the newest known
70
+ -- next timestamp.
71
+ rcall("ZADD", markerKey, nextTimestamp, "1")
72
+ end
73
+ end
74
+ --[[
75
+ Bake in the job id first 12 bits into the timestamp
76
+ to guarantee correct execution order of delayed jobs
77
+ (up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
78
+ WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
79
+ ]]
80
+ local function getDelayedScore(delayedKey, timestamp, delay)
81
+ local delayedTimestamp = (delay > 0 and (tonumber(timestamp) + delay)) or tonumber(timestamp)
82
+ local minScore = delayedTimestamp * 0x1000
83
+ local maxScore = (delayedTimestamp + 1 ) * 0x1000 - 1
84
+ local result = rcall("ZREVRANGEBYSCORE", delayedKey, maxScore,
85
+ minScore, "WITHSCORES","LIMIT", 0, 1)
86
+ if #result then
87
+ local currentMaxScore = tonumber(result[2])
88
+ if currentMaxScore ~= nil then
89
+ if currentMaxScore >= maxScore then
90
+ return maxScore, delayedTimestamp
91
+ else
92
+ return currentMaxScore + 1, delayedTimestamp
93
+ end
94
+ end
95
+ end
96
+ return minScore, delayedTimestamp
97
+ end
98
+ local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
99
+ maxEvents, markerKey, delay)
100
+ local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
101
+ rcall("ZADD", delayedKey, score, jobId)
102
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
103
+ "jobId", jobId, "delay", delayedTimestamp)
104
+ -- mark that a delayed job is available
105
+ addDelayMarkerIfNeeded(markerKey, delayedKey)
106
+ end
107
+ --[[
108
+ Function to add job considering priority.
109
+ ]]
34
110
  -- Includes
111
+ --[[
112
+ Add marker if needed when a job is available.
113
+ ]]
114
+ local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
115
+ if not isPausedOrMaxed then
116
+ rcall("ZADD", markerKey, 0, "0")
117
+ end
118
+ end
119
+ --[[
120
+ Function to get priority score.
121
+ ]]
122
+ local function getPriorityScore(priority, priorityCounterKey)
123
+ local prioCounter = rcall("INCR", priorityCounterKey)
124
+ return priority * 0x100000000 + prioCounter % 0x100000000
125
+ end
126
+ local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
127
+ isPausedOrMaxed)
128
+ local score = getPriorityScore(priority, priorityCounterKey)
129
+ rcall("ZADD", prioritizedKey, score, jobId)
130
+ addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
131
+ end
132
+ --[[
133
+ Function to get max events value or set by default 10000.
134
+ ]]
135
+ local function getOrSetMaxEvents(metaKey)
136
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
137
+ if not maxEvents then
138
+ maxEvents = 10000
139
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
140
+ end
141
+ return maxEvents
142
+ end
143
+ --[[
144
+ Function to check for the meta.paused key to decide if we are paused or not
145
+ (since an empty list and !EXISTS are not really the same).
146
+ ]]
147
+ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
148
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
149
+ if queueAttributes[1] then
150
+ return pausedKey, true
151
+ else
152
+ if queueAttributes[2] then
153
+ local activeCount = rcall("LLEN", activeKey)
154
+ if activeCount >= tonumber(queueAttributes[2]) then
155
+ return waitKey, true
156
+ else
157
+ return waitKey, false
158
+ end
159
+ end
160
+ end
161
+ return waitKey, false
162
+ end
35
163
  --[[
36
164
  Function to check for the meta.paused key to decide if we are paused or not
37
165
  (since an empty list and !EXISTS are not really the same).
@@ -70,14 +198,6 @@ end
70
198
  Function to add job in target list and add marker if needed.
71
199
  ]]
72
200
  -- Includes
73
- --[[
74
- Add marker if needed when a job is available.
75
- ]]
76
- local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
77
- if not isPausedOrMaxed then
78
- rcall("ZADD", markerKey, 0, "0")
79
- end
80
- end
81
201
  local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
82
202
  rcall(pushCmd, targetKey, jobId)
83
203
  addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
@@ -92,26 +212,6 @@ end
92
212
  local getJobKeyPrefix = function (jobKey, jobId)
93
213
  return string.sub(jobKey, 0, #jobKey - #jobId)
94
214
  end
95
- --[[
96
- Function to check for the meta.paused key to decide if we are paused or not
97
- (since an empty list and !EXISTS are not really the same).
98
- ]]
99
- local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
100
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
101
- if queueAttributes[1] then
102
- return pausedKey, true
103
- else
104
- if queueAttributes[2] then
105
- local activeCount = rcall("LLEN", activeKey)
106
- if activeCount >= tonumber(queueAttributes[2]) then
107
- return waitKey, true
108
- else
109
- return waitKey, false
110
- end
111
- end
112
- end
113
- return waitKey, false
114
- end
115
215
  local function moveParentToWait(parentPrefix, parentId, emitEvent)
116
216
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
117
217
  parentPrefix .. "wait", parentPrefix .. "paused")
@@ -192,6 +292,36 @@ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
192
292
  end
193
293
  removeJobKeys(jobKey)
194
294
  end
295
+ --[[
296
+ Function to store a job
297
+ ]]
298
+ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
299
+ parentKey, parentData, repeatJobKey)
300
+ local jsonOpts = cjson.encode(opts)
301
+ local delay = opts['delay'] or 0
302
+ local priority = opts['priority'] or 0
303
+ local debounceId = opts['de'] and opts['de']['id']
304
+ local optionalValues = {}
305
+ if parentKey ~= nil then
306
+ table.insert(optionalValues, "parentKey")
307
+ table.insert(optionalValues, parentKey)
308
+ table.insert(optionalValues, "parent")
309
+ table.insert(optionalValues, parentData)
310
+ end
311
+ if repeatJobKey ~= nil then
312
+ table.insert(optionalValues, "rjk")
313
+ table.insert(optionalValues, repeatJobKey)
314
+ end
315
+ if debounceId then
316
+ table.insert(optionalValues, "deid")
317
+ table.insert(optionalValues, debounceId)
318
+ end
319
+ rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
320
+ "timestamp", timestamp, "delay", delay, "priority", priority,
321
+ unpack(optionalValues))
322
+ rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
323
+ return delay, priority
324
+ end
195
325
  --[[
196
326
  Function to store a job scheduler
197
327
  ]]
@@ -233,11 +363,11 @@ end
233
363
  -- If we are overriding a repeatable job we must delete the delayed job for
234
364
  -- the next iteration.
235
365
  local schedulerKey = repeatKey .. ":" .. jobSchedulerId
366
+ local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
367
+ local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
236
368
  local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
237
369
  if prevMillis ~= false then
238
370
  local currentJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
239
- local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
240
- local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
241
371
  if rcall("EXISTS", nextDelayedJobKey) ~= 1 or currentJobId == nextDelayedJobId then
242
372
  if rcall("ZSCORE", delayedKey, currentJobId) ~= false then
243
373
  removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
@@ -259,11 +389,47 @@ if prevMillis ~= false then
259
389
  end
260
390
  end
261
391
  local schedulerOpts = cmsgpack.unpack(ARGV[2])
262
- return storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, ARGV[4], templateOpts)
392
+ storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, ARGV[4], templateOpts)
393
+ if rcall("EXISTS", nextDelayedJobKey) ~= 1 then
394
+ local eventsKey = KEYS[9]
395
+ local metaKey = KEYS[5]
396
+ local maxEvents = getOrSetMaxEvents(metaKey)
397
+ rcall("INCR", KEYS[8])
398
+ local delayedOpts = cmsgpack.unpack(ARGV[6])
399
+ local delay, priority = storeJob(eventsKey, nextDelayedJobKey, nextDelayedJobId, schedulerOpts['name'], ARGV[4],
400
+ delayedOpts, ARGV[7], nil, nil, jobSchedulerId)
401
+ if delay ~= 0 then
402
+ addDelayedJob(nextDelayedJobId, delayedKey, eventsKey,
403
+ ARGV[7], maxEvents, KEYS[7], delay)
404
+ else
405
+ local isPaused = isQueuePaused(KEYS[5])
406
+ -- Standard or priority add
407
+ if priority == 0 then
408
+ if isPaused then
409
+ -- LIFO or FIFO
410
+ local pushCmd = delayedOpts['lifo'] and 'RPUSH' or 'LPUSH'
411
+ rcall(pushCmd, KEYS[4], nextDelayedJobId)
412
+ else
413
+ -- LIFO or FIFO
414
+ local pushCmd = delayedOpts['lifo'] and 'RPUSH' or 'LPUSH'
415
+ rcall(pushCmd, KEYS[3], nextDelayedJobId)
416
+ end
417
+ else
418
+ -- Priority add
419
+ addJobWithPriority(KEYS[7], KEYS[6], priority, nextDelayedJobId, KEYS[10], isPaused)
420
+ end
421
+ -- Emit waiting event
422
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting", "jobId", nextDelayedJobId)
423
+ end
424
+ if ARGV[9] ~= "" then
425
+ rcall("HSET", ARGV[9], "nrjid", nextDelayedJobId)
426
+ end
427
+ return nextDelayedJobId .. "" -- convert to string
428
+ end
263
429
  `;
264
430
  exports.addJobScheduler = {
265
431
  name: 'addJobScheduler',
266
432
  content,
267
- keys: 6,
433
+ keys: 10,
268
434
  };
269
- //# sourceMappingURL=addJobScheduler-6.js.map
435
+ //# sourceMappingURL=addJobScheduler-10.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"addJobScheduler-10.js","sourceRoot":"","sources":["../../../src/scripts/addJobScheduler-10.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyaf,CAAC;AACW,QAAA,eAAe,GAAG;IAC7B,IAAI,EAAE,iBAAiB;IACvB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
@@ -71,12 +71,12 @@ end
71
71
  Function to get max events value or set by default 10000.
72
72
  ]]
73
73
  local function getOrSetMaxEvents(metaKey)
74
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
75
- if not maxEvents then
76
- maxEvents = 10000
77
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
78
- end
79
- return maxEvents
74
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
75
+ if not maxEvents then
76
+ maxEvents = 10000
77
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
78
+ end
79
+ return maxEvents
80
80
  end
81
81
  --[[
82
82
  Function to handle the case when job is duplicated.
@@ -133,12 +133,12 @@ end
133
133
  Function to get max events value or set by default 10000.
134
134
  ]]
135
135
  local function getOrSetMaxEvents(metaKey)
136
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
137
- if not maxEvents then
138
- maxEvents = 10000
139
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
140
- end
141
- return maxEvents
136
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
137
+ if not maxEvents then
138
+ maxEvents = 10000
139
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
140
+ end
141
+ return maxEvents
142
142
  end
143
143
  --[[
144
144
  Function to handle the case when job is duplicated.
@@ -97,12 +97,12 @@ end
97
97
  Function to get max events value or set by default 10000.
98
98
  ]]
99
99
  local function getOrSetMaxEvents(metaKey)
100
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
101
- if not maxEvents then
102
- maxEvents = 10000
103
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
104
- end
105
- return maxEvents
100
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
101
+ if not maxEvents then
102
+ maxEvents = 10000
103
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
104
+ end
105
+ return maxEvents
106
106
  end
107
107
  --[[
108
108
  Function to check for the meta.paused key to decide if we are paused or not
@@ -73,12 +73,12 @@ end
73
73
  Function to get max events value or set by default 10000.
74
74
  ]]
75
75
  local function getOrSetMaxEvents(metaKey)
76
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
77
- if not maxEvents then
78
- maxEvents = 10000
79
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
80
- end
81
- return maxEvents
76
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
77
+ if not maxEvents then
78
+ maxEvents = 10000
79
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
80
+ end
81
+ return maxEvents
82
82
  end
83
83
  if rcall("EXISTS", ARGV[4]) == 1 then
84
84
  local jobId = ARGV[3]
@@ -6,13 +6,14 @@ const content = `--[[
6
6
  Input:
7
7
  KEYS[1] set key,
8
8
  KEYS[2] events stream key
9
- KEYS[3] job schedulers key
9
+ KEYS[3] repeat key
10
10
  ARGV[1] jobKey prefix
11
11
  ARGV[2] timestamp
12
12
  ARGV[3] limit the number of jobs to be removed. 0 is unlimited
13
13
  ARGV[4] set name, can be any of 'wait', 'active', 'paused', 'delayed', 'completed', or 'failed'
14
14
  ]]
15
15
  local rcall = redis.call
16
+ local repeatKey = KEYS[3]
16
17
  local rangeStart = 0
17
18
  local rangeEnd = -1
18
19
  local limit = tonumber(ARGV[3])
@@ -305,7 +306,7 @@ local function cleanSet(
305
306
  local jobKey = jobKeyPrefix .. job
306
307
  -- Extract a Job Scheduler Id from jobId ("repeat:job-scheduler-id:millis")
307
308
  -- and check if it is in the scheduled jobs
308
- if not isJobSchedulerJob(job, jobKey, jobSchedulersKey) then
309
+ if not (jobSchedulersKey and isJobSchedulerJob(job, jobKey, jobSchedulersKey)) then
309
310
  if isFinished then
310
311
  removeJob(job, true, jobKeyPrefix, true --[[remove debounce key]] )
311
312
  deletedCount = deletedCount + 1
@@ -331,20 +332,21 @@ end
331
332
  local result
332
333
  if ARGV[4] == "active" then
333
334
  result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], false --[[ hasFinished ]],
334
- KEYS[3])
335
+ repeatKey)
335
336
  elseif ARGV[4] == "delayed" then
336
337
  rangeEnd = "+inf"
337
338
  result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
338
- {"processedOn", "timestamp"}, false --[[ hasFinished ]], KEYS[3])
339
+ {"processedOn", "timestamp"}, false --[[ hasFinished ]], repeatKey)
339
340
  elseif ARGV[4] == "prioritized" then
340
341
  rangeEnd = "+inf"
341
342
  result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
342
- {"timestamp"}, false --[[ hasFinished ]])
343
+ {"timestamp"}, false --[[ hasFinished ]], repeatKey)
343
344
  elseif ARGV[4] == "wait" or ARGV[4] == "paused" then
344
345
  result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], true --[[ hasFinished ]],
345
- KEYS[3])
346
+ repeatKey)
346
347
  else
347
348
  rangeEnd = ARGV[2]
349
+ -- No need to pass repeat key as in that moment job won't be related to a job scheduler
348
350
  result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
349
351
  {"finishedOn"}, true --[[ hasFinished ]])
350
352
  end
@@ -1 +1 @@
1
- {"version":3,"file":"cleanJobsInSet-3.js","sourceRoot":"","sources":["../../../src/scripts/cleanJobsInSet-3.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6Vf,CAAC;AACW,QAAA,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"cleanJobsInSet-3.js","sourceRoot":"","sources":["../../../src/scripts/cleanJobsInSet-3.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+Vf,CAAC;AACW,QAAA,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -2,7 +2,7 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  const tslib_1 = require("tslib");
4
4
  tslib_1.__exportStar(require("./addDelayedJob-6"), exports);
5
- tslib_1.__exportStar(require("./addJobScheduler-6"), exports);
5
+ tslib_1.__exportStar(require("./addJobScheduler-10"), exports);
6
6
  tslib_1.__exportStar(require("./addLog-2"), exports);
7
7
  tslib_1.__exportStar(require("./addParentJob-4"), exports);
8
8
  tslib_1.__exportStar(require("./addPrioritizedJob-8"), exports);
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/scripts/index.ts"],"names":[],"mappings":";;;AAAA,4DAAkC;AAClC,8DAAoC;AACpC,qDAA2B;AAC3B,2DAAiC;AACjC,gEAAsC;AACtC,+DAAqC;AACrC,6DAAmC;AACnC,0DAAgC;AAChC,6DAAmC;AACnC,6DAAmC;AACnC,oDAA0B;AAC1B,yDAA+B;AAC/B,0DAAgC;AAChC,wDAA8B;AAC9B,mEAAyC;AACzC,8DAAoC;AACpC,wDAA8B;AAC9B,8DAAoC;AACpC,uDAA6B;AAC7B,yDAA+B;AAC/B,yDAA+B;AAC/B,0DAAgC;AAChC,sDAA4B;AAC5B,sEAA4C;AAC5C,6DAAmC;AACnC,oEAA0C;AAC1C,4DAAkC;AAClC,4DAAkC;AAClC,8DAAoC;AACpC,oEAA0C;AAC1C,yDAA+B;AAC/B,uDAA6B;AAC7B,oDAA0B;AAC1B,sDAA4B;AAC5B,0DAAgC;AAChC,oEAA0C;AAC1C,wDAA8B;AAC9B,iEAAuC;AACvC,+DAAqC;AACrC,2DAAiC;AACjC,wDAA8B;AAC9B,6DAAmC;AACnC,yDAA+B;AAC/B,iEAAuC;AACvC,6DAAmC;AACnC,wEAA8C"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/scripts/index.ts"],"names":[],"mappings":";;;AAAA,4DAAkC;AAClC,+DAAqC;AACrC,qDAA2B;AAC3B,2DAAiC;AACjC,gEAAsC;AACtC,+DAAqC;AACrC,6DAAmC;AACnC,0DAAgC;AAChC,6DAAmC;AACnC,6DAAmC;AACnC,oDAA0B;AAC1B,yDAA+B;AAC/B,0DAAgC;AAChC,wDAA8B;AAC9B,mEAAyC;AACzC,8DAAoC;AACpC,wDAA8B;AAC9B,8DAAoC;AACpC,uDAA6B;AAC7B,yDAA+B;AAC/B,yDAA+B;AAC/B,0DAAgC;AAChC,sDAA4B;AAC5B,sEAA4C;AAC5C,6DAAmC;AACnC,oEAA0C;AAC1C,4DAAkC;AAClC,4DAAkC;AAClC,8DAAoC;AACpC,oEAA0C;AAC1C,yDAA+B;AAC/B,uDAA6B;AAC7B,oDAA0B;AAC1B,sDAA4B;AAC5B,0DAAgC;AAChC,oEAA0C;AAC1C,wDAA8B;AAC9B,iEAAuC;AACvC,+DAAqC;AACrC,2DAAiC;AACjC,wDAA8B;AAC9B,6DAAmC;AACnC,yDAA+B;AAC/B,iEAAuC;AACvC,6DAAmC;AACnC,wEAA8C"}
@@ -48,12 +48,12 @@ end
48
48
  Function to get max events value or set by default 10000.
49
49
  ]]
50
50
  local function getOrSetMaxEvents(metaKey)
51
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
52
- if not maxEvents then
53
- maxEvents = 10000
54
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
55
- end
56
- return maxEvents
51
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
52
+ if not maxEvents then
53
+ maxEvents = 10000
54
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
55
+ end
56
+ return maxEvents
57
57
  end
58
58
  --[[
59
59
  Function to check for the meta.paused key to decide if we are paused or not
@@ -52,12 +52,12 @@ end
52
52
  Function to get max events value or set by default 10000.
53
53
  ]]
54
54
  local function getOrSetMaxEvents(metaKey)
55
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
56
- if not maxEvents then
57
- maxEvents = 10000
58
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
59
- end
60
- return maxEvents
55
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
56
+ if not maxEvents then
57
+ maxEvents = 10000
58
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
59
+ end
60
+ return maxEvents
61
61
  end
62
62
  --[[
63
63
  Function to check for the meta.paused key to decide if we are paused or not
@@ -361,8 +361,11 @@ local function removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestam
361
361
  end
362
362
  end
363
363
  end
364
- local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
365
- if rcall("ZREM", parentQueueKey .. ":waiting-children", parentId) == 1 then
364
+ local function moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
365
+ local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
366
+ local parentDelayedKey = parentQueueKey .. ":delayed"
367
+ if rcall("ZSCORE", parentWaitingChildrenKey, parentId) ~= false then
368
+ rcall("ZREM", parentWaitingChildrenKey, parentId)
366
369
  local parentQueuePrefix = parentQueueKey .. ":"
367
370
  local parentFailedKey = parentQueueKey .. ":failed"
368
371
  rcall("ZADD", parentFailedKey, timestamp, parentId)
@@ -398,6 +401,43 @@ local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey,
398
401
  local parentRawOpts = jobAttributes[3]
399
402
  local parentOpts = cjson.decode(parentRawOpts)
400
403
  removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
404
+ elseif rcall("ZSCORE", parentDelayedKey, parentId) ~= false then
405
+ rcall("ZREM", parentDelayedKey, parentId)
406
+ local parentQueuePrefix = parentQueueKey .. ":"
407
+ local parentFailedKey = parentQueueKey .. ":failed"
408
+ rcall("ZADD", parentFailedKey, timestamp, parentId)
409
+ local failedReason = "child " .. jobIdKey .. " failed"
410
+ rcall("HMSET", parentKey, "failedReason", failedReason, "finishedOn", timestamp)
411
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason",
412
+ failedReason, "prev", "delayed")
413
+ local jobAttributes = rcall("HMGET", parentKey, "parent", "deid", "opts")
414
+ removeDeduplicationKeyIfNeeded(parentQueueKey .. ":", jobAttributes[2])
415
+ if jobAttributes[1] then
416
+ local parentData = cjson.decode(jobAttributes[1])
417
+ if parentData['fpof'] then
418
+ moveParentFromWaitingChildrenToFailed(
419
+ parentData['queueKey'],
420
+ parentData['queueKey'] .. ':' .. parentData['id'],
421
+ parentData['id'],
422
+ parentKey,
423
+ timestamp
424
+ )
425
+ elseif parentData['idof'] or parentData['rdof'] then
426
+ local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
427
+ local grandParentDependenciesSet = grandParentKey .. ":dependencies"
428
+ if rcall("SREM", grandParentDependenciesSet, parentKey) == 1 then
429
+ moveParentToWaitIfNeeded(parentData['queueKey'], grandParentDependenciesSet,
430
+ grandParentKey, parentData['id'], timestamp)
431
+ if parentData['idof'] then
432
+ local grandParentFailedSet = grandParentKey .. ":failed"
433
+ rcall("HSET", grandParentFailedSet, parentKey, failedReason)
434
+ end
435
+ end
436
+ end
437
+ end
438
+ local parentRawOpts = jobAttributes[3]
439
+ local parentOpts = cjson.decode(parentRawOpts)
440
+ removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
401
441
  end
402
442
  end
403
443
  --[[
@@ -408,12 +448,12 @@ end
408
448
  Function to get max events value or set by default 10000.
409
449
  ]]
410
450
  local function getOrSetMaxEvents(metaKey)
411
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
412
- if not maxEvents then
413
- maxEvents = 10000
414
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
415
- end
416
- return maxEvents
451
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
452
+ if not maxEvents then
453
+ maxEvents = 10000
454
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
455
+ end
456
+ return maxEvents
417
457
  end
418
458
  local function trimEvents(metaKey, eventStreamKey)
419
459
  local maxEvents = getOrSetMaxEvents(metaKey)
@@ -476,7 +516,7 @@ if (#stalling > 0) then
476
516
  rcall("XADD", eventStreamKey, "*", "event",
477
517
  "failed", "jobId", jobId, 'prev', 'active',
478
518
  'failedReason', failedReason)
479
- if rawParentData ~= false then
519
+ if rawParentData then
480
520
  if opts['fpof'] then
481
521
  local parentData = cjson.decode(rawParentData)
482
522
  moveParentFromWaitingChildrenToFailed(
@@ -1 +1 @@
1
- {"version":3,"file":"moveStalledJobsToWait-9.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yBA6gBS,CAAC;AACb,QAAA,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"moveStalledJobsToWait-9.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yBAqjBS,CAAC;AACb,QAAA,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -80,12 +80,12 @@ end
80
80
  Function to get max events value or set by default 10000.
81
81
  ]]
82
82
  local function getOrSetMaxEvents(metaKey)
83
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
84
- if not maxEvents then
85
- maxEvents = 10000
86
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
87
- end
88
- return maxEvents
83
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
84
+ if not maxEvents then
85
+ maxEvents = 10000
86
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
87
+ end
88
+ return maxEvents
89
89
  end
90
90
  local function removeLock(jobKey, stalledKey, token, jobId)
91
91
  if token ~= "0" then