bullmq 5.1.3 → 5.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (103) hide show
  1. package/dist/cjs/classes/scripts.js +1 -0
  2. package/dist/cjs/classes/scripts.js.map +1 -1
  3. package/dist/cjs/commands/addStandardJob-7.lua +4 -8
  4. package/dist/cjs/commands/changeDelay-3.lua +1 -0
  5. package/dist/cjs/commands/changePriority-6.lua +0 -1
  6. package/dist/cjs/commands/includes/addBaseMarkerIfNeeded.lua +9 -0
  7. package/dist/cjs/commands/includes/addDelayMarkerIfNeeded.lua +3 -1
  8. package/dist/cjs/commands/includes/addJobInTargetList.lua +11 -0
  9. package/dist/cjs/commands/includes/addJobWithPriority.lua +5 -3
  10. package/dist/cjs/commands/includes/moveParentToWaitIfNeeded.lua +6 -4
  11. package/dist/cjs/commands/includes/prepareJobForProcessing.lua +3 -39
  12. package/dist/cjs/commands/includes/promoteDelayedJobs.lua +2 -4
  13. package/dist/cjs/commands/moveJobsToWait-7.lua +2 -3
  14. package/dist/cjs/commands/moveStalledJobsToWait-9.lua +150 -0
  15. package/dist/cjs/commands/moveToActive-11.lua +6 -5
  16. package/dist/cjs/commands/moveToFinished-14.lua +13 -11
  17. package/dist/cjs/commands/pause-7.lua +1 -0
  18. package/dist/cjs/commands/promote-8.lua +2 -2
  19. package/dist/cjs/commands/retryJob-10.lua +1 -0
  20. package/dist/cjs/scripts/addDelayedJob-6.js +21 -7
  21. package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
  22. package/dist/cjs/scripts/addParentJob-4.js +25 -11
  23. package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
  24. package/dist/cjs/scripts/addPrioritizedJob-7.js +21 -7
  25. package/dist/cjs/scripts/addPrioritizedJob-7.js.map +1 -1
  26. package/dist/cjs/scripts/addStandardJob-7.js +46 -36
  27. package/dist/cjs/scripts/addStandardJob-7.js.map +1 -1
  28. package/dist/cjs/scripts/changeDelay-3.js +1 -0
  29. package/dist/cjs/scripts/changeDelay-3.js.map +1 -1
  30. package/dist/cjs/scripts/changePriority-6.js +10 -14
  31. package/dist/cjs/scripts/changePriority-6.js.map +1 -1
  32. package/dist/cjs/scripts/index.js +1 -1
  33. package/dist/cjs/scripts/moveJobsToWait-7.js +9 -3
  34. package/dist/cjs/scripts/moveJobsToWait-7.js.map +1 -1
  35. package/dist/cjs/scripts/{moveStalledJobsToWait-8.js → moveStalledJobsToWait-9.js} +109 -106
  36. package/dist/cjs/scripts/{moveStalledJobsToWait-8.js.map → moveStalledJobsToWait-9.js.map} +1 -1
  37. package/dist/cjs/scripts/moveToActive-11.js +28 -50
  38. package/dist/cjs/scripts/moveToActive-11.js.map +1 -1
  39. package/dist/cjs/scripts/moveToDelayed-7.js +1 -1
  40. package/dist/cjs/scripts/moveToFinished-14.js +47 -70
  41. package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
  42. package/dist/cjs/scripts/pause-7.js +2 -1
  43. package/dist/cjs/scripts/pause-7.js.map +1 -1
  44. package/dist/cjs/scripts/promote-8.js +19 -5
  45. package/dist/cjs/scripts/promote-8.js.map +1 -1
  46. package/dist/cjs/scripts/retryJob-10.js +20 -7
  47. package/dist/cjs/scripts/retryJob-10.js.map +1 -1
  48. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  49. package/dist/esm/classes/scripts.js +1 -0
  50. package/dist/esm/classes/scripts.js.map +1 -1
  51. package/dist/esm/commands/addStandardJob-7.lua +4 -8
  52. package/dist/esm/commands/changeDelay-3.lua +1 -0
  53. package/dist/esm/commands/changePriority-6.lua +0 -1
  54. package/dist/esm/commands/includes/addBaseMarkerIfNeeded.lua +9 -0
  55. package/dist/esm/commands/includes/addDelayMarkerIfNeeded.lua +3 -1
  56. package/dist/esm/commands/includes/addJobInTargetList.lua +11 -0
  57. package/dist/esm/commands/includes/addJobWithPriority.lua +5 -3
  58. package/dist/esm/commands/includes/moveParentToWaitIfNeeded.lua +6 -4
  59. package/dist/esm/commands/includes/prepareJobForProcessing.lua +3 -39
  60. package/dist/esm/commands/includes/promoteDelayedJobs.lua +2 -4
  61. package/dist/esm/commands/moveJobsToWait-7.lua +2 -3
  62. package/dist/esm/commands/moveStalledJobsToWait-9.lua +150 -0
  63. package/dist/esm/commands/moveToActive-11.lua +6 -5
  64. package/dist/esm/commands/moveToFinished-14.lua +13 -11
  65. package/dist/esm/commands/pause-7.lua +1 -0
  66. package/dist/esm/commands/promote-8.lua +2 -2
  67. package/dist/esm/commands/retryJob-10.lua +1 -0
  68. package/dist/esm/scripts/addDelayedJob-6.js +21 -7
  69. package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
  70. package/dist/esm/scripts/addParentJob-4.js +25 -11
  71. package/dist/esm/scripts/addParentJob-4.js.map +1 -1
  72. package/dist/esm/scripts/addPrioritizedJob-7.js +21 -7
  73. package/dist/esm/scripts/addPrioritizedJob-7.js.map +1 -1
  74. package/dist/esm/scripts/addStandardJob-7.js +46 -36
  75. package/dist/esm/scripts/addStandardJob-7.js.map +1 -1
  76. package/dist/esm/scripts/changeDelay-3.js +1 -0
  77. package/dist/esm/scripts/changeDelay-3.js.map +1 -1
  78. package/dist/esm/scripts/changePriority-6.js +10 -14
  79. package/dist/esm/scripts/changePriority-6.js.map +1 -1
  80. package/dist/esm/scripts/index.d.ts +1 -1
  81. package/dist/esm/scripts/index.js +1 -1
  82. package/dist/esm/scripts/moveJobsToWait-7.js +9 -3
  83. package/dist/esm/scripts/moveJobsToWait-7.js.map +1 -1
  84. package/dist/esm/scripts/{moveStalledJobsToWait-8.js → moveStalledJobsToWait-9.js} +109 -106
  85. package/dist/esm/scripts/{moveStalledJobsToWait-8.js.map → moveStalledJobsToWait-9.js.map} +1 -1
  86. package/dist/esm/scripts/moveToActive-11.js +28 -50
  87. package/dist/esm/scripts/moveToActive-11.js.map +1 -1
  88. package/dist/esm/scripts/moveToDelayed-7.js +1 -1
  89. package/dist/esm/scripts/moveToFinished-14.js +47 -70
  90. package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
  91. package/dist/esm/scripts/pause-7.js +2 -1
  92. package/dist/esm/scripts/pause-7.js.map +1 -1
  93. package/dist/esm/scripts/promote-8.js +19 -5
  94. package/dist/esm/scripts/promote-8.js.map +1 -1
  95. package/dist/esm/scripts/retryJob-10.js +20 -7
  96. package/dist/esm/scripts/retryJob-10.js.map +1 -1
  97. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  98. package/package.json +1 -1
  99. package/dist/cjs/commands/includes/checkStalledJobs.lua +0 -139
  100. package/dist/cjs/commands/moveStalledJobsToWait-8.lua +0 -24
  101. package/dist/esm/commands/includes/checkStalledJobs.lua +0 -139
  102. package/dist/esm/commands/moveStalledJobsToWait-8.lua +0 -24
  103. /package/dist/esm/scripts/{moveStalledJobsToWait-8.d.ts → moveStalledJobsToWait-9.d.ts} +0 -0
@@ -1 +1 @@
1
- {"version":3,"file":"moveStalledJobsToWait-8.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAoSf,CAAC;AACF,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"moveStalledJobsToWait-9.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-9.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yBAuSS,CAAC;AAC1B,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -30,6 +30,7 @@ const content = `--[[
30
30
  local rcall = redis.call
31
31
  local waitKey = KEYS[1]
32
32
  local activeKey = KEYS[2]
33
+ local eventStreamKey = KEYS[4]
33
34
  local rateLimiterKey = KEYS[6]
34
35
  local delayedKey = KEYS[7]
35
36
  local opts = cmsgpack.unpack(ARGV[3])
@@ -88,50 +89,15 @@ end
88
89
  --[[
89
90
  Function to move job from wait state to active.
90
91
  Input:
91
- keys[1] wait key
92
- keys[2] active key
93
- keys[3] prioritized key
94
- keys[4] stream events key
95
- keys[5] stalled key
96
- -- Rate limiting
97
- keys[6] rate limiter key
98
- keys[7] delayed key
99
- keys[8] paused key
100
- keys[9] meta key
101
- keys[10] pc priority counter
102
92
  opts - token - lock token
103
93
  opts - lockDuration
104
94
  opts - limiter
105
95
  ]]
106
- -- Includes
107
- --[[
108
- Function to push back job considering priority in front of same prioritized jobs.
109
- ]]
110
- local function pushBackJobWithPriority(prioritizedKey, priority, jobId)
111
- -- in order to put it at front of same prioritized jobs
112
- -- we consider prioritized counter as 0
113
- local score = priority * 0x100000000
114
- rcall("ZADD", prioritizedKey, score, jobId)
115
- end
116
- local function prepareJobForProcessing(keys, keyPrefix, targetKey, jobId, processedOn,
117
- maxJobs, expireTime, opts)
96
+ local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey,
97
+ jobId, processedOn, maxJobs, opts)
118
98
  local jobKey = keyPrefix .. jobId
119
99
  -- Check if we need to perform rate limiting.
120
100
  if maxJobs then
121
- local rateLimiterKey = keys[6];
122
- -- check if we exceeded rate limit, we need to remove the job and return expireTime
123
- if expireTime > 0 then
124
- -- remove from active queue and add back to the wait list
125
- rcall("LREM", keys[2], 1, jobId)
126
- local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
127
- if priority == 0 then
128
- rcall("RPUSH", targetKey, jobId)
129
- else
130
- pushBackJobWithPriority(keys[3], priority, jobId)
131
- end
132
- -- Return when we can process more jobs
133
- return {0, 0, expireTime, 0}
134
- end
135
101
  local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
136
102
  if jobCounter == 1 then
137
103
  local limiterDuration = opts['limiter'] and opts['limiter']['duration']
@@ -144,7 +110,7 @@ local function prepareJobForProcessing(keys, keyPrefix, targetKey, jobId, proces
144
110
  if opts['token'] ~= "0" then
145
111
  rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
146
112
  end
147
- rcall("XADD", keys[4], "*", "event", "active", "jobId", jobId, "prev", "waiting")
113
+ rcall("XADD", eventStreamKey, "*", "event", "active", "jobId", jobId, "prev", "waiting")
148
114
  rcall("HSET", jobKey, "processedOn", processedOn)
149
115
  rcall("HINCRBY", jobKey, "ats", 1)
150
116
  return {rcall("HGETALL", jobKey), jobId, 0, 0} -- get job data
@@ -156,16 +122,31 @@ end
156
122
  'waiting'
157
123
  ]]
158
124
  -- Includes
125
+ --[[
126
+ Function to add job in target list and add marker if needed.
127
+ ]]
128
+ -- Includes
129
+ --[[
130
+ Add marker if needed when a job is available.
131
+ ]]
132
+ local function addBaseMarkerIfNeeded(markerKey, isPaused)
133
+ if not isPaused then
134
+ rcall("ZADD", markerKey, 0, "0")
135
+ end
136
+ end
137
+ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
138
+ rcall(pushCmd, targetKey, jobId)
139
+ addBaseMarkerIfNeeded(markerKey, isPaused)
140
+ end
159
141
  --[[
160
142
  Function to add job considering priority.
161
143
  ]]
144
+ -- Includes
162
145
  local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
163
146
  local prioCounter = rcall("INCR", priorityCounterKey)
164
147
  local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
165
148
  rcall("ZADD", prioritizedKey, score, jobId)
166
- if not isPaused then
167
- rcall("ZADD", markerKey, 0, "0")
168
- end
149
+ addBaseMarkerIfNeeded(markerKey, isPaused)
169
150
  end
170
151
  -- Try to get as much as 1000 jobs at once
171
152
  local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedKey,
@@ -179,10 +160,7 @@ local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedK
179
160
  tonumber(rcall("HGET", jobKey, "priority")) or 0
180
161
  if priority == 0 then
181
162
  -- LIFO or FIFO
182
- rcall("LPUSH", targetKey, jobId)
183
- if not isPaused then
184
- rcall("ZADD", markerKey, 0, "0")
185
- end
163
+ addJobInTargetList(targetKey, markerKey, "LPUSH", isPaused, jobId)
186
164
  else
187
165
  addJobWithPriority(markerKey, prioritizedKey, priority,
188
166
  jobId, priorityCounterKey, isPaused)
@@ -197,7 +175,7 @@ end
197
175
  local target, paused = getTargetQueueList(KEYS[9], waitKey, KEYS[8])
198
176
  -- Check if there are delayed jobs that we can move to wait.
199
177
  local markerKey = KEYS[11]
200
- promoteDelayedJobs(delayedKey, markerKey, target, KEYS[3], KEYS[4], ARGV[1],
178
+ promoteDelayedJobs(delayedKey, markerKey, target, KEYS[3], eventStreamKey, ARGV[1],
201
179
  ARGV[2], KEYS[10], paused)
202
180
  local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
203
181
  local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
@@ -213,13 +191,13 @@ if jobId and string.sub(jobId, 1, 2) == "0:" then
213
191
  jobId = rcall("RPOPLPUSH", waitKey, activeKey)
214
192
  end
215
193
  if jobId then
216
- return prepareJobForProcessing(KEYS, ARGV[1], target, jobId, ARGV[2],
217
- maxJobs, expireTime, opts)
194
+ return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
195
+ maxJobs, opts)
218
196
  else
219
197
  jobId = moveJobFromPriorityToActive(KEYS[3], activeKey, KEYS[10])
220
198
  if jobId then
221
- return prepareJobForProcessing(KEYS, ARGV[1], target, jobId, ARGV[2],
222
- maxJobs, expireTime, opts)
199
+ return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
200
+ maxJobs, opts)
223
201
  end
224
202
  end
225
203
  -- Return the timestamp for the next delayed job if any.
@@ -1 +1 @@
1
- {"version":3,"file":"moveToActive-11.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-11.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAoOf,CAAC;AACF,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
1
+ {"version":3,"file":"moveToActive-11.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-11.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8Mf,CAAC;AACF,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
@@ -26,7 +26,7 @@ local rcall = redis.call
26
26
  -- Includes
27
27
  --[[
28
28
  Add delay marker if needed.
29
- ]]
29
+ ]]
30
30
  -- Includes
31
31
  --[[
32
32
  Function to return the next delayed job timestamp.
@@ -113,6 +113,17 @@ local function getNextDelayedTimestamp(delayedKey)
113
113
  return nextTimestamp
114
114
  end
115
115
  end
116
+ --[[
117
+ Function to check for the meta.paused key to decide if we are paused or not
118
+ (since an empty list and !EXISTS are not really the same).
119
+ ]]
120
+ local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
121
+ if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
122
+ return waitKey, false
123
+ else
124
+ return pausedKey, true
125
+ end
126
+ end
116
127
  --[[
117
128
  Function to move job from prioritized state to active.
118
129
  ]]
@@ -128,50 +139,15 @@ end
128
139
  --[[
129
140
  Function to move job from wait state to active.
130
141
  Input:
131
- keys[1] wait key
132
- keys[2] active key
133
- keys[3] prioritized key
134
- keys[4] stream events key
135
- keys[5] stalled key
136
- -- Rate limiting
137
- keys[6] rate limiter key
138
- keys[7] delayed key
139
- keys[8] paused key
140
- keys[9] meta key
141
- keys[10] pc priority counter
142
142
  opts - token - lock token
143
143
  opts - lockDuration
144
144
  opts - limiter
145
145
  ]]
146
- -- Includes
147
- --[[
148
- Function to push back job considering priority in front of same prioritized jobs.
149
- ]]
150
- local function pushBackJobWithPriority(prioritizedKey, priority, jobId)
151
- -- in order to put it at front of same prioritized jobs
152
- -- we consider prioritized counter as 0
153
- local score = priority * 0x100000000
154
- rcall("ZADD", prioritizedKey, score, jobId)
155
- end
156
- local function prepareJobForProcessing(keys, keyPrefix, targetKey, jobId, processedOn,
157
- maxJobs, expireTime, opts)
146
+ local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey,
147
+ jobId, processedOn, maxJobs, opts)
158
148
  local jobKey = keyPrefix .. jobId
159
149
  -- Check if we need to perform rate limiting.
160
150
  if maxJobs then
161
- local rateLimiterKey = keys[6];
162
- -- check if we exceeded rate limit, we need to remove the job and return expireTime
163
- if expireTime > 0 then
164
- -- remove from active queue and add back to the wait list
165
- rcall("LREM", keys[2], 1, jobId)
166
- local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
167
- if priority == 0 then
168
- rcall("RPUSH", targetKey, jobId)
169
- else
170
- pushBackJobWithPriority(keys[3], priority, jobId)
171
- end
172
- -- Return when we can process more jobs
173
- return {0, 0, expireTime, 0}
174
- end
175
151
  local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
176
152
  if jobCounter == 1 then
177
153
  local limiterDuration = opts['limiter'] and opts['limiter']['duration']
@@ -184,7 +160,7 @@ local function prepareJobForProcessing(keys, keyPrefix, targetKey, jobId, proces
184
160
  if opts['token'] ~= "0" then
185
161
  rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
186
162
  end
187
- rcall("XADD", keys[4], "*", "event", "active", "jobId", jobId, "prev", "waiting")
163
+ rcall("XADD", eventStreamKey, "*", "event", "active", "jobId", jobId, "prev", "waiting")
188
164
  rcall("HSET", jobKey, "processedOn", processedOn)
189
165
  rcall("HINCRBY", jobKey, "ats", 1)
190
166
  return {rcall("HGETALL", jobKey), jobId, 0, 0} -- get job data
@@ -199,7 +175,7 @@ end
199
175
  -- Includes
200
176
  --[[
201
177
  Add delay marker if needed.
202
- ]]
178
+ ]]
203
179
  -- Includes
204
180
  local function addDelayMarkerIfNeeded(markerKey, delayedKey)
205
181
  local nextTimestamp = getNextDelayedTimestamp(delayedKey)
@@ -210,33 +186,37 @@ local function addDelayMarkerIfNeeded(markerKey, delayedKey)
210
186
  end
211
187
  end
212
188
  --[[
213
- Function to check for the meta.paused key to decide if we are paused or not
214
- (since an empty list and !EXISTS are not really the same).
189
+ Function to add job in target list and add marker if needed.
215
190
  ]]
216
- local function isQueuePaused(queueMetaKey)
217
- return rcall("HEXISTS", queueMetaKey, "paused") == 1
191
+ -- Includes
192
+ --[[
193
+ Add marker if needed when a job is available.
194
+ ]]
195
+ local function addBaseMarkerIfNeeded(markerKey, isPaused)
196
+ if not isPaused then
197
+ rcall("ZADD", markerKey, 0, "0")
198
+ end
199
+ end
200
+ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
201
+ rcall(pushCmd, targetKey, jobId)
202
+ addBaseMarkerIfNeeded(markerKey, isPaused)
218
203
  end
219
204
  --[[
220
205
  Function to add job considering priority.
221
206
  ]]
207
+ -- Includes
222
208
  local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
223
209
  local prioCounter = rcall("INCR", priorityCounterKey)
224
210
  local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
225
211
  rcall("ZADD", prioritizedKey, score, jobId)
226
- if not isPaused then
227
- rcall("ZADD", markerKey, 0, "0")
228
- end
212
+ addBaseMarkerIfNeeded(markerKey, isPaused)
229
213
  end
230
214
  --[[
231
215
  Function to check for the meta.paused key to decide if we are paused or not
232
216
  (since an empty list and !EXISTS are not really the same).
233
217
  ]]
234
- local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
235
- if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
236
- return waitKey, false
237
- else
238
- return pausedKey, true
239
- end
218
+ local function isQueuePaused(queueMetaKey)
219
+ return rcall("HEXISTS", queueMetaKey, "paused") == 1
240
220
  end
241
221
  local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
242
222
  parentKey, parentId, timestamp)
@@ -261,11 +241,10 @@ local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
261
241
  addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
262
242
  else
263
243
  if priority == 0 then
264
- local parentTarget, _paused =
244
+ local parentTarget, isParentPaused =
265
245
  getTargetQueueList(parentMetaKey, parentWaitKey,
266
246
  parentPausedKey)
267
- rcall("RPUSH", parentTarget, parentId)
268
- rcall("ZADD", parentMarkerKey, 0, "0")
247
+ addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPaused, parentId)
269
248
  else
270
249
  local isPaused = isQueuePaused(parentMetaKey)
271
250
  addJobWithPriority(parentMarkerKey,
@@ -325,10 +304,7 @@ local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedK
325
304
  tonumber(rcall("HGET", jobKey, "priority")) or 0
326
305
  if priority == 0 then
327
306
  -- LIFO or FIFO
328
- rcall("LPUSH", targetKey, jobId)
329
- if not isPaused then
330
- rcall("ZADD", markerKey, 0, "0")
331
- end
307
+ addJobInTargetList(targetKey, markerKey, "LPUSH", isPaused, jobId)
332
308
  else
333
309
  addJobWithPriority(markerKey, prioritizedKey, priority,
334
310
  jobId, priorityCounterKey, isPaused)
@@ -540,9 +516,10 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
540
516
  -- Remove from active list (if not active we shall return error)
541
517
  local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
542
518
  if (numRemovedElements < 1) then return -3 end
519
+ local eventStreamKey = KEYS[4]
543
520
  local metaKey = KEYS[9]
544
521
  -- Trim events before emiting them to avoid trimming events emitted in this script
545
- trimEvents(metaKey, KEYS[4])
522
+ trimEvents(metaKey, eventStreamKey)
546
523
  -- If job has a parent we need to
547
524
  -- 1) remove this job id from parents dependencies
548
525
  -- 2) move the job Id to parent "processed" set
@@ -596,11 +573,11 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
596
573
  removeParentDependencyKey(jobIdKey, false, parentKey)
597
574
  end
598
575
  end
599
- rcall("XADD", KEYS[4], "*", "event", ARGV[5], "jobId", jobId, ARGV[3],
576
+ rcall("XADD", eventStreamKey, "*", "event", ARGV[5], "jobId", jobId, ARGV[3],
600
577
  ARGV[4])
601
578
  if ARGV[5] == "failed" then
602
579
  if tonumber(attemptsMade) >= tonumber(attempts) then
603
- rcall("XADD", KEYS[4], "*", "event", "retries-exhausted", "jobId",
580
+ rcall("XADD", eventStreamKey, "*", "event", "retries-exhausted", "jobId",
604
581
  jobId, "attemptsMade", attemptsMade)
605
582
  end
606
583
  end
@@ -613,7 +590,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
613
590
  if (ARGV[6] == "1") then
614
591
  local target, paused = getTargetQueueList(metaKey, KEYS[1], KEYS[8])
615
592
  -- Check if there are delayed jobs that can be promoted
616
- promoteDelayedJobs(KEYS[7], KEYS[14], target, KEYS[3], KEYS[4], ARGV[7],
593
+ promoteDelayedJobs(KEYS[7], KEYS[14], target, KEYS[3], eventStreamKey, ARGV[7],
617
594
  timestamp, KEYS[10], paused)
618
595
  local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
619
596
  -- Check if we are rate limited first.
@@ -631,20 +608,20 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
631
608
  if jobId == "0:0" then
632
609
  jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2],
633
610
  KEYS[10])
634
- return prepareJobForProcessing(KEYS, ARGV[7], target, jobId,
611
+ return prepareJobForProcessing(ARGV[7], KEYS[6], eventStreamKey, jobId,
635
612
  timestamp, maxJobs,
636
- expireTime, opts)
613
+ opts)
637
614
  end
638
615
  else
639
- return prepareJobForProcessing(KEYS, ARGV[7], target, jobId,
640
- timestamp, maxJobs, expireTime,
616
+ return prepareJobForProcessing(ARGV[7], KEYS[6], eventStreamKey, jobId,
617
+ timestamp, maxJobs,
641
618
  opts)
642
619
  end
643
620
  else
644
621
  jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
645
622
  if jobId then
646
- return prepareJobForProcessing(KEYS, ARGV[7], target, jobId,
647
- timestamp, maxJobs, expireTime,
623
+ return prepareJobForProcessing(ARGV[7], KEYS[6], eventStreamKey, jobId,
624
+ timestamp, maxJobs,
648
625
  opts)
649
626
  end
650
627
  end
@@ -662,7 +639,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
662
639
  if activeLen == 0 then
663
640
  local prioritizedLen = rcall("ZCARD", KEYS[3])
664
641
  if prioritizedLen == 0 then
665
- rcall("XADD", KEYS[4], "*", "event", "drained")
642
+ rcall("XADD", eventStreamKey, "*", "event", "drained")
666
643
  end
667
644
  end
668
645
  end
@@ -1 +1 @@
1
- {"version":3,"file":"moveToFinished-14.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-14.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgqBf,CAAC;AACF,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
1
+ {"version":3,"file":"moveToFinished-14.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-14.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyoBf,CAAC;AACF,MAAM,CAAC,MAAM,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
@@ -16,7 +16,7 @@ local rcall = redis.call
16
16
  -- Includes
17
17
  --[[
18
18
  Add delay marker if needed.
19
- ]]
19
+ ]]
20
20
  -- Includes
21
21
  --[[
22
22
  Function to return the next delayed job timestamp.
@@ -41,6 +41,7 @@ local function addDelayMarkerIfNeeded(markerKey, delayedKey)
41
41
  end
42
42
  local markerKey = KEYS[7]
43
43
  local hasJobs = rcall("EXISTS", KEYS[1]) == 1
44
+ --TODO: check this logic to be reused when changing a delay
44
45
  if hasJobs then rcall("RENAME", KEYS[1], KEYS[2]) end
45
46
  if ARGV[1] == "paused" then
46
47
  rcall("HSET", KEYS[3], "paused", 1)
@@ -1 +1 @@
1
- {"version":3,"file":"pause-7.js","sourceRoot":"","sources":["../../../src/scripts/pause-7.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyDf,CAAC;AACF,MAAM,CAAC,MAAM,KAAK,GAAG;IACnB,IAAI,EAAE,OAAO;IACb,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"pause-7.js","sourceRoot":"","sources":["../../../src/scripts/pause-7.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0Df,CAAC;AACF,MAAM,CAAC,MAAM,KAAK,GAAG;IACnB,IAAI,EAAE,OAAO;IACb,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -20,16 +20,31 @@ const content = `--[[
20
20
  local rcall = redis.call
21
21
  local jobId = ARGV[2]
22
22
  -- Includes
23
+ --[[
24
+ Function to add job in target list and add marker if needed.
25
+ ]]
26
+ -- Includes
27
+ --[[
28
+ Add marker if needed when a job is available.
29
+ ]]
30
+ local function addBaseMarkerIfNeeded(markerKey, isPaused)
31
+ if not isPaused then
32
+ rcall("ZADD", markerKey, 0, "0")
33
+ end
34
+ end
35
+ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
36
+ rcall(pushCmd, targetKey, jobId)
37
+ addBaseMarkerIfNeeded(markerKey, isPaused)
38
+ end
23
39
  --[[
24
40
  Function to add job considering priority.
25
41
  ]]
42
+ -- Includes
26
43
  local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
27
44
  local prioCounter = rcall("INCR", priorityCounterKey)
28
45
  local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
29
46
  rcall("ZADD", prioritizedKey, score, jobId)
30
- if not isPaused then
31
- rcall("ZADD", markerKey, 0, "0")
32
- end
47
+ addBaseMarkerIfNeeded(markerKey, isPaused)
33
48
  end
34
49
  --[[
35
50
  Function to check for the meta.paused key to decide if we are paused or not
@@ -54,8 +69,7 @@ if rcall("ZREM", KEYS[1], jobId) == 1 then
54
69
  if marker and string.sub(marker, 1, 2) == "0:" then rcall("LPOP", target) end
55
70
  if priority == 0 then
56
71
  -- LIFO or FIFO
57
- rcall("LPUSH", target, jobId)
58
- if not paused then rcall("ZADD", KEYS[8], 0, "0") end
72
+ addJobInTargetList(target, KEYS[8], "LPUSH", paused, jobId)
59
73
  else
60
74
  addJobWithPriority(KEYS[8], KEYS[5], priority, jobId, KEYS[6], paused)
61
75
  end
@@ -1 +1 @@
1
- {"version":3,"file":"promote-8.js","sourceRoot":"","sources":["../../../src/scripts/promote-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqEf,CAAC;AACF,MAAM,CAAC,MAAM,OAAO,GAAG;IACrB,IAAI,EAAE,SAAS;IACf,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"promote-8.js","sourceRoot":"","sources":["../../../src/scripts/promote-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAmFf,CAAC;AACF,MAAM,CAAC,MAAM,OAAO,GAAG;IACrB,IAAI,EAAE,SAAS;IACf,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -28,13 +28,20 @@ local rcall = redis.call
28
28
  --[[
29
29
  Function to add job considering priority.
30
30
  ]]
31
+ -- Includes
32
+ --[[
33
+ Add marker if needed when a job is available.
34
+ ]]
35
+ local function addBaseMarkerIfNeeded(markerKey, isPaused)
36
+ if not isPaused then
37
+ rcall("ZADD", markerKey, 0, "0")
38
+ end
39
+ end
31
40
  local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
32
41
  local prioCounter = rcall("INCR", priorityCounterKey)
33
42
  local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
34
43
  rcall("ZADD", prioritizedKey, score, jobId)
35
- if not isPaused then
36
- rcall("ZADD", markerKey, 0, "0")
37
- end
44
+ addBaseMarkerIfNeeded(markerKey, isPaused)
38
45
  end
39
46
  --[[
40
47
  Function to get max events value or set by default 10000.
@@ -65,6 +72,14 @@ end
65
72
  'waiting'
66
73
  ]]
67
74
  -- Includes
75
+ --[[
76
+ Function to add job in target list and add marker if needed.
77
+ ]]
78
+ -- Includes
79
+ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
80
+ rcall(pushCmd, targetKey, jobId)
81
+ addBaseMarkerIfNeeded(markerKey, isPaused)
82
+ end
68
83
  -- Try to get as much as 1000 jobs at once
69
84
  local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedKey,
70
85
  eventStreamKey, prefix, timestamp, priorityCounterKey, isPaused)
@@ -77,10 +92,7 @@ local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedK
77
92
  tonumber(rcall("HGET", jobKey, "priority")) or 0
78
93
  if priority == 0 then
79
94
  -- LIFO or FIFO
80
- rcall("LPUSH", targetKey, jobId)
81
- if not isPaused then
82
- rcall("ZADD", markerKey, 0, "0")
83
- end
95
+ addJobInTargetList(targetKey, markerKey, "LPUSH", isPaused, jobId)
84
96
  else
85
97
  addJobWithPriority(markerKey, prioritizedKey, priority,
86
98
  jobId, priorityCounterKey, isPaused)
@@ -111,6 +123,7 @@ if rcall("EXISTS", KEYS[4]) == 1 then
111
123
  -- Standard or priority add
112
124
  if priority == 0 then
113
125
  rcall(ARGV[3], target, ARGV[4])
126
+ -- TODO: check if we need to add marker in this case too
114
127
  else
115
128
  addJobWithPriority(markerKey, KEYS[8], priority, ARGV[4], KEYS[9], paused)
116
129
  end
@@ -1 +1 @@
1
- {"version":3,"file":"retryJob-10.js","sourceRoot":"","sources":["../../../src/scripts/retryJob-10.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA6Hf,CAAC;AACF,MAAM,CAAC,MAAM,QAAQ,GAAG;IACtB,IAAI,EAAE,UAAU;IAChB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
1
+ {"version":3,"file":"retryJob-10.js","sourceRoot":"","sources":["../../../src/scripts/retryJob-10.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0If,CAAC;AACF,MAAM,CAAC,MAAM,QAAQ,GAAG;IACtB,IAAI,EAAE,UAAU;IAChB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}