bullmq 5.1.3 → 5.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/dist/cjs/classes/scripts.js +1 -0
  2. package/dist/cjs/classes/scripts.js.map +1 -1
  3. package/dist/cjs/commands/addStandardJob-7.lua +4 -8
  4. package/dist/cjs/commands/changeDelay-3.lua +1 -0
  5. package/dist/cjs/commands/changePriority-6.lua +0 -1
  6. package/dist/cjs/commands/includes/addBaseMarkerIfNeeded.lua +9 -0
  7. package/dist/cjs/commands/includes/addDelayMarkerIfNeeded.lua +3 -1
  8. package/dist/cjs/commands/includes/addJobInTargetList.lua +11 -0
  9. package/dist/cjs/commands/includes/addJobWithPriority.lua +5 -3
  10. package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +6 -4
  11. package/dist/cjs/commands/includes/prepareJobForProcessing.lua +3 -39
  12. package/dist/cjs/commands/includes/promoteDelayedJobs.lua +2 -4
  13. package/dist/cjs/commands/moveJobsToWait-7.lua +2 -3
  14. package/dist/cjs/commands/moveStalledJobsToWait-9.lua +150 -0
  15. package/dist/cjs/commands/moveToActive-11.lua +6 -5
  16. package/dist/cjs/commands/moveToFinished-14.lua +13 -11
  17. package/dist/cjs/commands/pause-7.lua +1 -0
  18. package/dist/cjs/commands/promote-8.lua +2 -2
  19. package/dist/cjs/commands/retryJob-10.lua +1 -0
  20. package/dist/cjs/scripts/addDelayedJob-6.js +21 -7
  21. package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
  22. package/dist/cjs/scripts/addParentJob-4.js +25 -11
  23. package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
  24. package/dist/cjs/scripts/addPrioritizedJob-7.js +21 -7
  25. package/dist/cjs/scripts/addPrioritizedJob-7.js.map +1 -1
  26. package/dist/cjs/scripts/addStandardJob-7.js +46 -36
  27. package/dist/cjs/scripts/addStandardJob-7.js.map +1 -1
  28. package/dist/cjs/scripts/changeDelay-3.js +1 -0
  29. package/dist/cjs/scripts/changeDelay-3.js.map +1 -1
  30. package/dist/cjs/scripts/changePriority-6.js +10 -14
  31. package/dist/cjs/scripts/changePriority-6.js.map +1 -1
  32. package/dist/cjs/scripts/index.js +1 -1
  33. package/dist/cjs/scripts/moveJobsToWait-7.js +9 -3
  34. package/dist/cjs/scripts/moveJobsToWait-7.js.map +1 -1
  35. package/dist/cjs/scripts/{moveStalledJobsToWait-8.js → moveStalledJobsToWait-9.js} +109 -106
  36. package/dist/cjs/scripts/{moveStalledJobsToWait-8.js.map → moveStalledJobsToWait-9.js.map} +1 -1
  37. package/dist/cjs/scripts/moveToActive-11.js +28 -50
  38. package/dist/cjs/scripts/moveToActive-11.js.map +1 -1
  39. package/dist/cjs/scripts/moveToDelayed-7.js +1 -1
  40. package/dist/cjs/scripts/moveToFinished-14.js +47 -70
  41. package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
  42. package/dist/cjs/scripts/pause-7.js +2 -1
  43. package/dist/cjs/scripts/pause-7.js.map +1 -1
  44. package/dist/cjs/scripts/promote-8.js +19 -5
  45. package/dist/cjs/scripts/promote-8.js.map +1 -1
  46. package/dist/cjs/scripts/retryJob-10.js +20 -7
  47. package/dist/cjs/scripts/retryJob-10.js.map +1 -1
  48. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  49. package/dist/esm/classes/scripts.js +1 -0
  50. package/dist/esm/classes/scripts.js.map +1 -1
  51. package/dist/esm/commands/addStandardJob-7.lua +4 -8
  52. package/dist/esm/commands/changeDelay-3.lua +1 -0
  53. package/dist/esm/commands/changePriority-6.lua +0 -1
  54. package/dist/esm/commands/includes/addBaseMarkerIfNeeded.lua +9 -0
  55. package/dist/esm/commands/includes/addDelayMarkerIfNeeded.lua +3 -1
  56. package/dist/esm/commands/includes/addJobInTargetList.lua +11 -0
  57. package/dist/esm/commands/includes/addJobWithPriority.lua +5 -3
  58. package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +6 -4
  59. package/dist/esm/commands/includes/prepareJobForProcessing.lua +3 -39
  60. package/dist/esm/commands/includes/promoteDelayedJobs.lua +2 -4
  61. package/dist/esm/commands/moveJobsToWait-7.lua +2 -3
  62. package/dist/esm/commands/moveStalledJobsToWait-9.lua +150 -0
  63. package/dist/esm/commands/moveToActive-11.lua +6 -5
  64. package/dist/esm/commands/moveToFinished-14.lua +13 -11
  65. package/dist/esm/commands/pause-7.lua +1 -0
  66. package/dist/esm/commands/promote-8.lua +2 -2
  67. package/dist/esm/commands/retryJob-10.lua +1 -0
  68. package/dist/esm/scripts/addDelayedJob-6.js +21 -7
  69. package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
  70. package/dist/esm/scripts/addParentJob-4.js +25 -11
  71. package/dist/esm/scripts/addParentJob-4.js.map +1 -1
  72. package/dist/esm/scripts/addPrioritizedJob-7.js +21 -7
  73. package/dist/esm/scripts/addPrioritizedJob-7.js.map +1 -1
  74. package/dist/esm/scripts/addStandardJob-7.js +46 -36
  75. package/dist/esm/scripts/addStandardJob-7.js.map +1 -1
  76. package/dist/esm/scripts/changeDelay-3.js +1 -0
  77. package/dist/esm/scripts/changeDelay-3.js.map +1 -1
  78. package/dist/esm/scripts/changePriority-6.js +10 -14
  79. package/dist/esm/scripts/changePriority-6.js.map +1 -1
  80. package/dist/esm/scripts/index.d.ts +1 -1
  81. package/dist/esm/scripts/index.js +1 -1
  82. package/dist/esm/scripts/moveJobsToWait-7.js +9 -3
  83. package/dist/esm/scripts/moveJobsToWait-7.js.map +1 -1
  84. package/dist/esm/scripts/{moveStalledJobsToWait-8.js → moveStalledJobsToWait-9.js} +109 -106
  85. package/dist/esm/scripts/{moveStalledJobsToWait-8.js.map → moveStalledJobsToWait-9.js.map} +1 -1
  86. package/dist/esm/scripts/moveToActive-11.js +28 -50
  87. package/dist/esm/scripts/moveToActive-11.js.map +1 -1
  88. package/dist/esm/scripts/moveToDelayed-7.js +1 -1
  89. package/dist/esm/scripts/moveToFinished-14.js +47 -70
  90. package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
  91. package/dist/esm/scripts/pause-7.js +2 -1
  92. package/dist/esm/scripts/pause-7.js.map +1 -1
  93. package/dist/esm/scripts/promote-8.js +19 -5
  94. package/dist/esm/scripts/promote-8.js.map +1 -1
  95. package/dist/esm/scripts/retryJob-10.js +20 -7
  96. package/dist/esm/scripts/retryJob-10.js.map +1 -1
  97. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  98. package/package.json +1 -1
  99. package/dist/cjs/commands/includes/checkStalledJobs.lua +0 -139
  100. package/dist/cjs/commands/moveStalledJobsToWait-8.lua +0 -24
  101. package/dist/esm/commands/includes/checkStalledJobs.lua +0 -139
  102. package/dist/esm/commands/moveStalledJobsToWait-8.lua +0 -24
  103. /package/dist/esm/scripts/{moveStalledJobsToWait-8.d.ts → moveStalledJobsToWait-9.d.ts} +0 -0
@@ -58,10 +58,11 @@ local parent = args[8]
58
58
  local parentData
59
59
 
60
60
  -- Includes
61
- --- @include "includes/storeJob"
62
- --- @include "includes/updateExistingJobsParent"
61
+ --- @include "includes/addJobInTargetList"
63
62
  --- @include "includes/getOrSetMaxEvents"
64
63
  --- @include "includes/getTargetQueueList"
64
+ --- @include "includes/storeJob"
65
+ --- @include "includes/updateExistingJobsParent"
65
66
 
66
67
  if parentKey ~= nil then
67
68
  if rcall("EXISTS", parentKey) ~= 1 then return -5 end
@@ -100,14 +101,9 @@ storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
100
101
 
101
102
  local target, paused = getTargetQueueList(metaKey, KEYS[1], KEYS[2])
102
103
 
103
- if not paused then
104
- -- mark that a job is available
105
- rcall("ZADD", KEYS[7], 0, "0")
106
- end
107
-
108
104
  -- LIFO or FIFO
109
105
  local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH'
110
- rcall(pushCmd, target, jobId)
106
+ addJobInTargetList(target, KEYS[7], pushCmd, paused, jobId)
111
107
 
112
108
  -- Emit waiting event
113
109
  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting",
@@ -31,6 +31,7 @@ if rcall("EXISTS", KEYS[2]) == 1 then
31
31
 
32
32
  rcall("HSET", KEYS[2], "delay", tonumber(ARGV[1]))
33
33
  rcall("ZADD", KEYS[1], score, jobId)
34
+ -- TODO: check if we need to evaluate a new marker
34
35
 
35
36
  rcall("XADD", KEYS[3], "*", "event", "delayed", "jobId", jobId, "delay", delayedTimestamp)
36
37
 
@@ -25,7 +25,6 @@ local rcall = redis.call
25
25
  -- Includes
26
26
  --- @include "includes/isQueuePaused"
27
27
  --- @include "includes/addJobWithPriority"
28
- --- @include "includes/getTargetQueueList"
29
28
 
30
29
  if rcall("EXISTS", jobKey) == 1 then
31
30
  local metaKey = KEYS[3]
@@ -0,0 +1,9 @@
1
+ --[[
2
+ Add marker if needed when a job is available.
3
+ ]]
4
+
5
+ local function addBaseMarkerIfNeeded(markerKey, isPaused)
6
+ if not isPaused then
7
+ rcall("ZADD", markerKey, 0, "0")
8
+ end
9
+ end
@@ -1,8 +1,10 @@
1
1
  --[[
2
2
  Add delay marker if needed.
3
- ]]
3
+ ]]
4
+
4
5
  -- Includes
5
6
  --- @include "getNextDelayedTimestamp"
7
+
6
8
  local function addDelayMarkerIfNeeded(markerKey, delayedKey)
7
9
  local nextTimestamp = getNextDelayedTimestamp(delayedKey)
8
10
  if nextTimestamp ~= nil then
@@ -0,0 +1,11 @@
1
+ --[[
2
+ Function to add job in target list and add marker if needed.
3
+ ]]
4
+
5
+ -- Includes
6
+ --- @include "addBaseMarkerIfNeeded"
7
+
8
+ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
9
+ rcall(pushCmd, targetKey, jobId)
10
+ addBaseMarkerIfNeeded(markerKey, isPaused)
11
+ end
@@ -1,11 +1,13 @@
1
1
  --[[
2
2
  Function to add job considering priority.
3
3
  ]]
4
+
5
+ -- Includes
6
+ --- @include "addBaseMarkerIfNeeded"
7
+
4
8
  local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
5
9
  local prioCounter = rcall("INCR", priorityCounterKey)
6
10
  local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
7
11
  rcall("ZADD", prioritizedKey, score, jobId)
8
- if not isPaused then
9
- rcall("ZADD", markerKey, 0, "0")
10
- end
12
+ addBaseMarkerIfNeeded(markerKey, isPaused)
11
13
  end
@@ -1,11 +1,14 @@
1
1
  --[[
2
2
  Validate and move parent to active if needed.
3
3
  ]]
4
+
4
5
  -- Includes
5
6
  --- @include "addDelayMarkerIfNeeded"
6
- --- @include "isQueuePaused"
7
+ --- @include "addJobInTargetList"
7
8
  --- @include "addJobWithPriority"
9
+ --- @include "isQueuePaused"
8
10
  --- @include "getTargetQueueList"
11
+
9
12
  local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
10
13
  parentKey, parentId, timestamp)
11
14
  local isParentActive = rcall("ZSCORE",
@@ -32,11 +35,10 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
32
35
  addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
33
36
  else
34
37
  if priority == 0 then
35
- local parentTarget, _paused =
38
+ local parentTarget, isParentPaused =
36
39
  getTargetQueueList(parentMetaKey, parentWaitKey,
37
40
  parentPausedKey)
38
- rcall("RPUSH", parentTarget, parentId)
39
- rcall("ZADD", parentMarkerKey, 0, "0")
41
+ addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPaused, parentId)
40
42
  else
41
43
  local isPaused = isQueuePaused(parentMetaKey)
42
44
  addJobWithPriority(parentMarkerKey,
@@ -2,53 +2,17 @@
2
2
  --[[
3
3
  Function to move job from wait state to active.
4
4
  Input:
5
- keys[1] wait key
6
- keys[2] active key
7
- keys[3] prioritized key
8
- keys[4] stream events key
9
- keys[5] stalled key
10
-
11
- -- Rate limiting
12
- keys[6] rate limiter key
13
- keys[7] delayed key
14
-
15
- keys[8] paused key
16
- keys[9] meta key
17
- keys[10] pc priority counter
18
-
19
5
  opts - token - lock token
20
6
  opts - lockDuration
21
7
  opts - limiter
22
8
  ]]
23
9
 
24
- -- Includes
25
- --- @include "pushBackJobWithPriority"
26
-
27
- local function prepareJobForProcessing(keys, keyPrefix, targetKey, jobId, processedOn,
28
- maxJobs, expireTime, opts)
10
+ local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey,
11
+ jobId, processedOn, maxJobs, opts)
29
12
  local jobKey = keyPrefix .. jobId
30
13
 
31
14
  -- Check if we need to perform rate limiting.
32
15
  if maxJobs then
33
- local rateLimiterKey = keys[6];
34
-
35
- -- check if we exceeded rate limit, we need to remove the job and return expireTime
36
- if expireTime > 0 then
37
- -- remove from active queue and add back to the wait list
38
- rcall("LREM", keys[2], 1, jobId)
39
-
40
- local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
41
-
42
- if priority == 0 then
43
- rcall("RPUSH", targetKey, jobId)
44
- else
45
- pushBackJobWithPriority(keys[3], priority, jobId)
46
- end
47
-
48
- -- Return when we can process more jobs
49
- return {0, 0, expireTime, 0}
50
- end
51
-
52
16
  local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
53
17
 
54
18
  if jobCounter == 1 then
@@ -65,7 +29,7 @@ local function prepareJobForProcessing(keys, keyPrefix, targetKey, jobId, proces
65
29
  rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
66
30
  end
67
31
 
68
- rcall("XADD", keys[4], "*", "event", "active", "jobId", jobId, "prev", "waiting")
32
+ rcall("XADD", eventStreamKey, "*", "event", "active", "jobId", jobId, "prev", "waiting")
69
33
  rcall("HSET", jobKey, "processedOn", processedOn)
70
34
  rcall("HINCRBY", jobKey, "ats", 1)
71
35
 
@@ -7,6 +7,7 @@
7
7
  ]]
8
8
 
9
9
  -- Includes
10
+ --- @include "addJobInTargetList"
10
11
  --- @include "addJobWithPriority"
11
12
 
12
13
  -- Try to get as much as 1000 jobs at once
@@ -24,10 +25,7 @@ local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedK
24
25
 
25
26
  if priority == 0 then
26
27
  -- LIFO or FIFO
27
- rcall("LPUSH", targetKey, jobId)
28
- if not isPaused then
29
- rcall("ZADD", markerKey, 0, "0")
30
- end
28
+ addJobInTargetList(targetKey, markerKey, "LPUSH", isPaused, jobId)
31
29
  else
32
30
  addJobWithPriority(markerKey, prioritizedKey, priority,
33
31
  jobId, priorityCounterKey, isPaused)
@@ -26,6 +26,7 @@ local timestamp = tonumber(ARGV[2])
26
26
  local rcall = redis.call;
27
27
 
28
28
  -- Includes
29
+ --- @include "includes/addBaseMarkerIfNeeded"
29
30
  --- @include "includes/batches"
30
31
  --- @include "includes/getOrSetMaxEvents"
31
32
  --- @include "includes/getTargetQueueList"
@@ -61,9 +62,7 @@ if (#jobs > 0) then
61
62
  rcall("LPUSH", target, unpack(jobs, from, to))
62
63
  end
63
64
 
64
- if not paused then
65
- rcall("ZADD", KEYS[7], 0, "0")
66
- end
65
+ addBaseMarkerIfNeeded(KEYS[7], paused)
67
66
  end
68
67
 
69
68
  maxCount = maxCount - #jobs
@@ -0,0 +1,150 @@
1
+ --[[
2
+ Move stalled jobs to wait.
3
+
4
+ Input:
5
+ KEYS[1] 'stalled' (SET)
6
+ KEYS[2] 'wait', (LIST)
7
+ KEYS[3] 'active', (LIST)
8
+ KEYS[4] 'failed', (ZSET)
9
+ KEYS[5] 'stalled-check', (KEY)
10
+ KEYS[6] 'meta', (KEY)
11
+ KEYS[7] 'paused', (LIST)
12
+ KEYS[8] 'marker'
13
+ KEYS[9] 'event stream' (STREAM)
14
+
15
+ ARGV[1] Max stalled job count
16
+ ARGV[2] queue.toKey('')
17
+ ARGV[3] timestamp
18
+ ARGV[4] max check time
19
+
20
+ Events:
21
+ 'stalled' with stalled job id.
22
+ ]]
23
+
24
+ local rcall = redis.call
25
+
26
+ -- Includes
27
+ --- @include "includes/addJobInTargetList"
28
+ --- @include "includes/batches"
29
+ --- @include "includes/getTargetQueueList"
30
+ --- @include "includes/removeJob"
31
+ --- @include "includes/removeJobsByMaxAge"
32
+ --- @include "includes/removeJobsByMaxCount"
33
+ --- @include "includes/trimEvents"
34
+
35
+ local stalledKey = KEYS[1]
36
+ local waitKey = KEYS[2]
37
+ local activeKey = KEYS[3]
38
+ local failedKey = KEYS[4]
39
+ local stalledCheckKey = KEYS[5]
40
+ local metaKey = KEYS[6]
41
+ local pausedKey = KEYS[7]
42
+ local markerKey = KEYS[8]
43
+ local eventStreamKey = KEYS[9]
44
+ local maxStalledJobCount = ARGV[1]
45
+ local queueKeyPrefix = ARGV[2]
46
+ local timestamp = ARGV[3]
47
+ local maxCheckTime = ARGV[4]
48
+
49
+ if rcall("EXISTS", stalledCheckKey) == 1 then return {{}, {}} end
50
+
51
+ rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
52
+
53
+ -- Trim events before emiting them to avoid trimming events emitted in this script
54
+ trimEvents(metaKey, eventStreamKey)
55
+
56
+ -- Move all stalled jobs to wait
57
+ local stalling = rcall('SMEMBERS', stalledKey)
58
+ local stalled = {}
59
+ local failed = {}
60
+ if (#stalling > 0) then
61
+ rcall('DEL', stalledKey)
62
+
63
+ local MAX_STALLED_JOB_COUNT = tonumber(maxStalledJobCount)
64
+
65
+ -- Remove from active list
66
+ for i, jobId in ipairs(stalling) do
67
+ -- Markers in waitlist DEPRECATED in v5: Remove in v6.
68
+ if string.sub(jobId, 1, 2) == "0:" then
69
+ -- If the jobId is a delay marker ID we just remove it.
70
+ rcall("LREM", activeKey, 1, jobId)
71
+ else
72
+ local jobKey = queueKeyPrefix .. jobId
73
+
74
+ -- Check that the lock is also missing, then we can handle this job as really stalled.
75
+ if (rcall("EXISTS", jobKey .. ":lock") == 0) then
76
+ -- Remove from the active queue.
77
+ local removed = rcall("LREM", activeKey, 1, jobId)
78
+
79
+ if (removed > 0) then
80
+ -- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
81
+ local stalledCount =
82
+ rcall("HINCRBY", jobKey, "stalledCounter", 1)
83
+ if (stalledCount > MAX_STALLED_JOB_COUNT) then
84
+ local rawOpts = rcall("HGET", jobKey, "opts")
85
+ local opts = cjson.decode(rawOpts)
86
+ local removeOnFailType = type(opts["removeOnFail"])
87
+ rcall("ZADD", failedKey, timestamp, jobId)
88
+ local failedReason =
89
+ "job stalled more than allowable limit"
90
+ rcall("HMSET", jobKey, "failedReason", failedReason,
91
+ "finishedOn", timestamp)
92
+ rcall("XADD", eventStreamKey, "*", "event",
93
+ "failed", "jobId", jobId, 'prev', 'active',
94
+ 'failedReason', failedReason)
95
+
96
+ if removeOnFailType == "number" then
97
+ removeJobsByMaxCount(opts["removeOnFail"],
98
+ failedKey, queueKeyPrefix)
99
+ elseif removeOnFailType == "boolean" then
100
+ if opts["removeOnFail"] then
101
+ removeJob(jobId, false, queueKeyPrefix)
102
+ rcall("ZREM", failedKey, jobId)
103
+ end
104
+ elseif removeOnFailType ~= "nil" then
105
+ local maxAge = opts["removeOnFail"]["age"]
106
+ local maxCount = opts["removeOnFail"]["count"]
107
+
108
+ if maxAge ~= nil then
109
+ removeJobsByMaxAge(timestamp, maxAge,
110
+ failedKey, queueKeyPrefix)
111
+ end
112
+
113
+ if maxCount ~= nil and maxCount > 0 then
114
+ removeJobsByMaxCount(maxCount, failedKey,
115
+ queueKeyPrefix)
116
+ end
117
+ end
118
+
119
+ table.insert(failed, jobId)
120
+ else
121
+ local target, isPaused=
122
+ getTargetQueueList(metaKey, waitKey, pausedKey)
123
+
124
+ -- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
125
+ addJobInTargetList(target, markerKey, "RPUSH", isPaused, jobId)
126
+
127
+ rcall("XADD", eventStreamKey, "*", "event",
128
+ "waiting", "jobId", jobId, 'prev', 'active')
129
+
130
+ -- Emit the stalled event
131
+ rcall("XADD", eventStreamKey, "*", "event",
132
+ "stalled", "jobId", jobId)
133
+ table.insert(stalled, jobId)
134
+ end
135
+ end
136
+ end
137
+ end
138
+ end
139
+ end
140
+
141
+ -- Mark potentially stalled jobs
142
+ local active = rcall('LRANGE', activeKey, 0, -1)
143
+
144
+ if (#active > 0) then
145
+ for from, to in batches(#active, 7000) do
146
+ rcall('SADD', stalledKey, unpack(active, from, to))
147
+ end
148
+ end
149
+
150
+ return {failed, stalled}
@@ -37,6 +37,7 @@
37
37
  local rcall = redis.call
38
38
  local waitKey = KEYS[1]
39
39
  local activeKey = KEYS[2]
40
+ local eventStreamKey = KEYS[4]
40
41
  local rateLimiterKey = KEYS[6]
41
42
  local delayedKey = KEYS[7]
42
43
  local opts = cmsgpack.unpack(ARGV[3])
@@ -53,7 +54,7 @@ local target, paused = getTargetQueueList(KEYS[9], waitKey, KEYS[8])
53
54
 
54
55
  -- Check if there are delayed jobs that we can move to wait.
55
56
  local markerKey = KEYS[11]
56
- promoteDelayedJobs(delayedKey, markerKey, target, KEYS[3], KEYS[4], ARGV[1],
57
+ promoteDelayedJobs(delayedKey, markerKey, target, KEYS[3], eventStreamKey, ARGV[1],
57
58
  ARGV[2], KEYS[10], paused)
58
59
 
59
60
  local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
@@ -75,13 +76,13 @@ if jobId and string.sub(jobId, 1, 2) == "0:" then
75
76
  end
76
77
 
77
78
  if jobId then
78
- return prepareJobForProcessing(KEYS, ARGV[1], target, jobId, ARGV[2],
79
- maxJobs, expireTime, opts)
79
+ return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
80
+ maxJobs, opts)
80
81
  else
81
82
  jobId = moveJobFromPriorityToActive(KEYS[3], activeKey, KEYS[10])
82
83
  if jobId then
83
- return prepareJobForProcessing(KEYS, ARGV[1], target, jobId, ARGV[2],
84
- maxJobs, expireTime, opts)
84
+ return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
85
+ maxJobs, opts)
85
86
  end
86
87
  end
87
88
 
@@ -57,6 +57,7 @@ local rcall = redis.call
57
57
  --- Includes
58
58
  --- @include "includes/collectMetrics"
59
59
  --- @include "includes/getNextDelayedTimestamp"
60
+ --- @include "includes/getTargetQueueList"
60
61
  --- @include "includes/moveJobFromPriorityToActive"
61
62
  --- @include "includes/prepareJobForProcessing"
62
63
  --- @include "includes/moveParentFromWaitingChildrenToFailed"
@@ -118,9 +119,10 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
118
119
 
119
120
  if (numRemovedElements < 1) then return -3 end
120
121
 
122
+ local eventStreamKey = KEYS[4]
121
123
  local metaKey = KEYS[9]
122
124
  -- Trim events before emiting them to avoid trimming events emitted in this script
123
- trimEvents(metaKey, KEYS[4])
125
+ trimEvents(metaKey, eventStreamKey)
124
126
 
125
127
  -- If job has a parent we need to
126
128
  -- 1) remove this job id from parents dependencies
@@ -182,12 +184,12 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
182
184
  end
183
185
  end
184
186
 
185
- rcall("XADD", KEYS[4], "*", "event", ARGV[5], "jobId", jobId, ARGV[3],
187
+ rcall("XADD", eventStreamKey, "*", "event", ARGV[5], "jobId", jobId, ARGV[3],
186
188
  ARGV[4])
187
189
 
188
190
  if ARGV[5] == "failed" then
189
191
  if tonumber(attemptsMade) >= tonumber(attempts) then
190
- rcall("XADD", KEYS[4], "*", "event", "retries-exhausted", "jobId",
192
+ rcall("XADD", eventStreamKey, "*", "event", "retries-exhausted", "jobId",
191
193
  jobId, "attemptsMade", attemptsMade)
192
194
  end
193
195
  end
@@ -204,7 +206,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
204
206
  local target, paused = getTargetQueueList(metaKey, KEYS[1], KEYS[8])
205
207
 
206
208
  -- Check if there are delayed jobs that can be promoted
207
- promoteDelayedJobs(KEYS[7], KEYS[14], target, KEYS[3], KEYS[4], ARGV[7],
209
+ promoteDelayedJobs(KEYS[7], KEYS[14], target, KEYS[3], eventStreamKey, ARGV[7],
208
210
  timestamp, KEYS[10], paused)
209
211
 
210
212
  local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
@@ -228,20 +230,20 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
228
230
  if jobId == "0:0" then
229
231
  jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2],
230
232
  KEYS[10])
231
- return prepareJobForProcessing(KEYS, ARGV[7], target, jobId,
233
+ return prepareJobForProcessing(ARGV[7], KEYS[6], eventStreamKey, jobId,
232
234
  timestamp, maxJobs,
233
- expireTime, opts)
235
+ opts)
234
236
  end
235
237
  else
236
- return prepareJobForProcessing(KEYS, ARGV[7], target, jobId,
237
- timestamp, maxJobs, expireTime,
238
+ return prepareJobForProcessing(ARGV[7], KEYS[6], eventStreamKey, jobId,
239
+ timestamp, maxJobs,
238
240
  opts)
239
241
  end
240
242
  else
241
243
  jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
242
244
  if jobId then
243
- return prepareJobForProcessing(KEYS, ARGV[7], target, jobId,
244
- timestamp, maxJobs, expireTime,
245
+ return prepareJobForProcessing(ARGV[7], KEYS[6], eventStreamKey, jobId,
246
+ timestamp, maxJobs,
245
247
  opts)
246
248
  end
247
249
  end
@@ -263,7 +265,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
263
265
  local prioritizedLen = rcall("ZCARD", KEYS[3])
264
266
 
265
267
  if prioritizedLen == 0 then
266
- rcall("XADD", KEYS[4], "*", "event", "drained")
268
+ rcall("XADD", eventStreamKey, "*", "event", "drained")
267
269
  end
268
270
  end
269
271
  end
@@ -22,6 +22,7 @@ local rcall = redis.call
22
22
 
23
23
  local markerKey = KEYS[7]
24
24
  local hasJobs = rcall("EXISTS", KEYS[1]) == 1
25
+ --TODO: check this logic to be reused when changing a delay
25
26
  if hasJobs then rcall("RENAME", KEYS[1], KEYS[2]) end
26
27
 
27
28
  if ARGV[1] == "paused" then
@@ -25,6 +25,7 @@ local rcall = redis.call
25
25
  local jobId = ARGV[2]
26
26
 
27
27
  -- Includes
28
+ --- @include "includes/addJobInTargetList"
28
29
  --- @include "includes/addJobWithPriority"
29
30
  --- @include "includes/getTargetQueueList"
30
31
 
@@ -42,8 +43,7 @@ if rcall("ZREM", KEYS[1], jobId) == 1 then
42
43
 
43
44
  if priority == 0 then
44
45
  -- LIFO or FIFO
45
- rcall("LPUSH", target, jobId)
46
- if not paused then rcall("ZADD", KEYS[8], 0, "0") end
46
+ addJobInTargetList(target, KEYS[8], "LPUSH", paused, jobId)
47
47
  else
48
48
  addJobWithPriority(KEYS[8], KEYS[5], priority, jobId, KEYS[6], paused)
49
49
  end
@@ -60,6 +60,7 @@ if rcall("EXISTS", KEYS[4]) == 1 then
60
60
  -- Standard or priority add
61
61
  if priority == 0 then
62
62
  rcall(ARGV[3], target, ARGV[4])
63
+ -- TODO: check if we need to add marker in this case too
63
64
  else
64
65
  addJobWithPriority(markerKey, KEYS[8], priority, ARGV[4], KEYS[9], paused)
65
66
  end
@@ -46,7 +46,7 @@ local parentData
46
46
  -- Includes
47
47
  --[[
48
48
  Add delay marker if needed.
49
- ]]
49
+ ]]
50
50
  -- Includes
51
51
  --[[
52
52
  Function to return the next delayed job timestamp.
@@ -127,16 +127,31 @@ end
127
127
  Validate and move parent to active if needed.
128
128
  ]]
129
129
  -- Includes
130
+ --[[
131
+ Function to add job in target list and add marker if needed.
132
+ ]]
133
+ -- Includes
134
+ --[[
135
+ Add marker if needed when a job is available.
136
+ ]]
137
+ local function addBaseMarkerIfNeeded(markerKey, isPaused)
138
+ if not isPaused then
139
+ rcall("ZADD", markerKey, 0, "0")
140
+ end
141
+ end
142
+ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
143
+ rcall(pushCmd, targetKey, jobId)
144
+ addBaseMarkerIfNeeded(markerKey, isPaused)
145
+ end
130
146
  --[[
131
147
  Function to add job considering priority.
132
148
  ]]
149
+ -- Includes
133
150
  local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
134
151
  local prioCounter = rcall("INCR", priorityCounterKey)
135
152
  local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
136
153
  rcall("ZADD", prioritizedKey, score, jobId)
137
- if not isPaused then
138
- rcall("ZADD", markerKey, 0, "0")
139
- end
154
+ addBaseMarkerIfNeeded(markerKey, isPaused)
140
155
  end
141
156
  --[[
142
157
  Function to check for the meta.paused key to decide if we are paused or not
@@ -172,11 +187,10 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
172
187
  addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
173
188
  else
174
189
  if priority == 0 then
175
- local parentTarget, _paused =
190
+ local parentTarget, isParentPaused =
176
191
  getTargetQueueList(parentMetaKey, parentWaitKey,
177
192
  parentPausedKey)
178
- rcall("RPUSH", parentTarget, parentId)
179
- rcall("ZADD", parentMarkerKey, 0, "0")
193
+ addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPaused, parentId)
180
194
  else
181
195
  local isPaused = isQueuePaused(parentMetaKey)
182
196
  addJobWithPriority(parentMarkerKey,
@@ -1 +1 @@
1
- {"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkQf,CAAC;AACF,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgRf,CAAC;AACF,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}