bullmq 2.2.0 → 2.3.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (234) hide show
  1. package/dist/cjs/classes/job.d.ts +1 -1
  2. package/dist/cjs/classes/redis-connection.d.ts +6 -2
  3. package/dist/cjs/classes/redis-connection.js +12 -5
  4. package/dist/cjs/classes/redis-connection.js.map +1 -1
  5. package/dist/cjs/scripts/addJob-8.d.ts +5 -0
  6. package/dist/{esm/commands/addJob-8.lua → cjs/scripts/addJob-8.js} +83 -31
  7. package/dist/cjs/scripts/addJob-8.js.map +1 -0
  8. package/dist/cjs/scripts/changeDelay-4.d.ts +5 -0
  9. package/dist/cjs/{commands/changeDelay-4.lua → scripts/changeDelay-4.js} +11 -11
  10. package/dist/cjs/scripts/changeDelay-4.js.map +1 -0
  11. package/dist/cjs/scripts/cleanJobsInSet-2.d.ts +5 -0
  12. package/dist/cjs/scripts/cleanJobsInSet-2.js +264 -0
  13. package/dist/cjs/scripts/cleanJobsInSet-2.js.map +1 -0
  14. package/dist/cjs/scripts/drain-4.d.ts +5 -0
  15. package/dist/cjs/{commands/includes/removeParentDependencyKey.lua → scripts/drain-4.js} +110 -10
  16. package/dist/cjs/scripts/drain-4.js.map +1 -0
  17. package/dist/cjs/scripts/extendLock-2.d.ts +5 -0
  18. package/dist/{esm/commands/extendLock-2.lua → cjs/scripts/extendLock-2.js} +11 -4
  19. package/dist/cjs/scripts/extendLock-2.js.map +1 -0
  20. package/dist/cjs/scripts/getState-7.d.ts +5 -0
  21. package/dist/{esm/commands/getState-7.lua → cjs/scripts/getState-7.js} +22 -12
  22. package/dist/cjs/scripts/getState-7.js.map +1 -0
  23. package/dist/cjs/scripts/getStateV2-7.d.ts +5 -0
  24. package/dist/cjs/{commands/getStateV2-7.lua → scripts/getStateV2-7.js} +11 -10
  25. package/dist/cjs/scripts/getStateV2-7.js.map +1 -0
  26. package/dist/cjs/scripts/index.d.ts +26 -0
  27. package/dist/cjs/scripts/index.js +30 -0
  28. package/dist/cjs/scripts/index.js.map +1 -0
  29. package/dist/cjs/scripts/isFinished-3.d.ts +5 -0
  30. package/dist/cjs/{commands/isFinished-3.lua → scripts/isFinished-3.js} +11 -10
  31. package/dist/cjs/scripts/isFinished-3.js.map +1 -0
  32. package/dist/cjs/scripts/isJobInList-1.d.ts +5 -0
  33. package/dist/cjs/scripts/isJobInList-1.js +32 -0
  34. package/dist/cjs/scripts/isJobInList-1.js.map +1 -0
  35. package/dist/cjs/scripts/moveStalledJobsToWait-8.d.ts +5 -0
  36. package/dist/cjs/scripts/moveStalledJobsToWait-8.js +291 -0
  37. package/dist/cjs/scripts/moveStalledJobsToWait-8.js.map +1 -0
  38. package/dist/cjs/scripts/moveToActive-9.d.ts +5 -0
  39. package/dist/cjs/scripts/moveToActive-9.js +207 -0
  40. package/dist/cjs/scripts/moveToActive-9.js.map +1 -0
  41. package/dist/cjs/scripts/moveToDelayed-5.d.ts +5 -0
  42. package/dist/cjs/{commands/moveToDelayed-5.lua → scripts/moveToDelayed-5.js} +11 -13
  43. package/dist/cjs/scripts/moveToDelayed-5.js.map +1 -0
  44. package/dist/cjs/scripts/moveToFinished-12.d.ts +5 -0
  45. package/dist/cjs/scripts/moveToFinished-12.js +538 -0
  46. package/dist/cjs/scripts/moveToFinished-12.js.map +1 -0
  47. package/dist/cjs/scripts/moveToWaitingChildren-4.d.ts +5 -0
  48. package/dist/cjs/{commands/moveToWaitingChildren-4.lua → scripts/moveToWaitingChildren-4.js} +11 -14
  49. package/dist/cjs/scripts/moveToWaitingChildren-4.js.map +1 -0
  50. package/dist/cjs/scripts/obliterate-2.d.ts +5 -0
  51. package/dist/cjs/scripts/obliterate-2.js +241 -0
  52. package/dist/cjs/scripts/obliterate-2.js.map +1 -0
  53. package/dist/cjs/scripts/pause-4.d.ts +5 -0
  54. package/dist/{esm/commands/pause-4.lua → cjs/scripts/pause-4.js} +11 -7
  55. package/dist/cjs/scripts/pause-4.js.map +1 -0
  56. package/dist/cjs/scripts/promote-6.d.ts +5 -0
  57. package/dist/cjs/{commands/promote-6.lua → scripts/promote-6.js} +22 -14
  58. package/dist/cjs/scripts/promote-6.js.map +1 -0
  59. package/dist/cjs/scripts/releaseLock-1.d.ts +5 -0
  60. package/dist/{esm/commands/releaseLock-1.lua → cjs/scripts/releaseLock-1.js} +11 -5
  61. package/dist/cjs/scripts/releaseLock-1.js.map +1 -0
  62. package/dist/cjs/scripts/removeJob-1.d.ts +5 -0
  63. package/dist/cjs/scripts/removeJob-1.js +206 -0
  64. package/dist/cjs/scripts/removeJob-1.js.map +1 -0
  65. package/dist/cjs/scripts/removeRepeatable-2.d.ts +5 -0
  66. package/dist/cjs/{commands/removeRepeatable-2.lua → scripts/removeRepeatable-2.js} +11 -7
  67. package/dist/cjs/scripts/removeRepeatable-2.js.map +1 -0
  68. package/dist/cjs/scripts/reprocessJob-4.d.ts +5 -0
  69. package/dist/{esm/commands/reprocessJob-4.lua → cjs/scripts/reprocessJob-4.js} +11 -5
  70. package/dist/cjs/scripts/reprocessJob-4.js.map +1 -0
  71. package/dist/cjs/scripts/retryJob-6.d.ts +5 -0
  72. package/dist/{esm/commands/retryJob-6.lua → cjs/scripts/retryJob-6.js} +22 -13
  73. package/dist/cjs/scripts/retryJob-6.js.map +1 -0
  74. package/dist/cjs/scripts/retryJobs-6.d.ts +5 -0
  75. package/dist/cjs/{commands/retryJobs-6.lua → scripts/retryJobs-6.js} +38 -15
  76. package/dist/cjs/scripts/retryJobs-6.js.map +1 -0
  77. package/dist/cjs/scripts/takeLock-1.d.ts +5 -0
  78. package/dist/cjs/{commands/takeLock-1.lua → scripts/takeLock-1.js} +11 -4
  79. package/dist/cjs/scripts/takeLock-1.js.map +1 -0
  80. package/dist/cjs/scripts/updateData-1.d.ts +5 -0
  81. package/dist/cjs/scripts/updateData-1.js +23 -0
  82. package/dist/cjs/scripts/updateData-1.js.map +1 -0
  83. package/dist/cjs/scripts/updateProgress-2.d.ts +5 -0
  84. package/dist/{esm/commands/updateProgress-2.lua → cjs/scripts/updateProgress-2.js} +11 -5
  85. package/dist/cjs/scripts/updateProgress-2.js.map +1 -0
  86. package/dist/esm/classes/job.d.ts +1 -1
  87. package/dist/esm/classes/redis-connection.d.ts +6 -2
  88. package/dist/esm/classes/redis-connection.js +12 -5
  89. package/dist/esm/classes/redis-connection.js.map +1 -1
  90. package/dist/esm/scripts/addJob-8.d.ts +5 -0
  91. package/dist/{cjs/commands/addJob-8.lua → esm/scripts/addJob-8.js} +80 -31
  92. package/dist/esm/scripts/addJob-8.js.map +1 -0
  93. package/dist/esm/scripts/changeDelay-4.d.ts +5 -0
  94. package/dist/esm/{commands/changeDelay-4.lua → scripts/changeDelay-4.js} +8 -11
  95. package/dist/esm/scripts/changeDelay-4.js.map +1 -0
  96. package/dist/esm/scripts/cleanJobsInSet-2.d.ts +5 -0
  97. package/dist/esm/scripts/cleanJobsInSet-2.js +261 -0
  98. package/dist/esm/scripts/cleanJobsInSet-2.js.map +1 -0
  99. package/dist/esm/scripts/drain-4.d.ts +5 -0
  100. package/dist/esm/{commands/includes/removeParentDependencyKey.lua → scripts/drain-4.js} +107 -10
  101. package/dist/esm/scripts/drain-4.js.map +1 -0
  102. package/dist/esm/scripts/extendLock-2.d.ts +5 -0
  103. package/dist/{cjs/commands/extendLock-2.lua → esm/scripts/extendLock-2.js} +8 -4
  104. package/dist/esm/scripts/extendLock-2.js.map +1 -0
  105. package/dist/esm/scripts/getState-7.d.ts +5 -0
  106. package/dist/{cjs/commands/getState-7.lua → esm/scripts/getState-7.js} +19 -12
  107. package/dist/esm/scripts/getState-7.js.map +1 -0
  108. package/dist/esm/scripts/getStateV2-7.d.ts +5 -0
  109. package/dist/esm/{commands/getStateV2-7.lua → scripts/getStateV2-7.js} +8 -10
  110. package/dist/esm/scripts/getStateV2-7.js.map +1 -0
  111. package/dist/esm/scripts/index.d.ts +26 -0
  112. package/dist/esm/scripts/index.js +27 -0
  113. package/dist/esm/scripts/index.js.map +1 -0
  114. package/dist/esm/scripts/isFinished-3.d.ts +5 -0
  115. package/dist/esm/{commands/isFinished-3.lua → scripts/isFinished-3.js} +8 -10
  116. package/dist/esm/scripts/isFinished-3.js.map +1 -0
  117. package/dist/esm/scripts/isJobInList-1.d.ts +5 -0
  118. package/dist/esm/scripts/isJobInList-1.js +29 -0
  119. package/dist/esm/scripts/isJobInList-1.js.map +1 -0
  120. package/dist/esm/scripts/moveStalledJobsToWait-8.d.ts +5 -0
  121. package/dist/esm/scripts/moveStalledJobsToWait-8.js +288 -0
  122. package/dist/esm/scripts/moveStalledJobsToWait-8.js.map +1 -0
  123. package/dist/esm/scripts/moveToActive-9.d.ts +5 -0
  124. package/dist/esm/scripts/moveToActive-9.js +204 -0
  125. package/dist/esm/scripts/moveToActive-9.js.map +1 -0
  126. package/dist/esm/scripts/moveToDelayed-5.d.ts +5 -0
  127. package/dist/esm/{commands/moveToDelayed-5.lua → scripts/moveToDelayed-5.js} +8 -13
  128. package/dist/esm/scripts/moveToDelayed-5.js.map +1 -0
  129. package/dist/esm/scripts/moveToFinished-12.d.ts +5 -0
  130. package/dist/esm/scripts/moveToFinished-12.js +535 -0
  131. package/dist/esm/scripts/moveToFinished-12.js.map +1 -0
  132. package/dist/esm/scripts/moveToWaitingChildren-4.d.ts +5 -0
  133. package/dist/esm/{commands/moveToWaitingChildren-4.lua → scripts/moveToWaitingChildren-4.js} +8 -14
  134. package/dist/esm/scripts/moveToWaitingChildren-4.js.map +1 -0
  135. package/dist/esm/scripts/obliterate-2.d.ts +5 -0
  136. package/dist/esm/scripts/obliterate-2.js +238 -0
  137. package/dist/esm/scripts/obliterate-2.js.map +1 -0
  138. package/dist/esm/scripts/pause-4.d.ts +5 -0
  139. package/dist/{cjs/commands/pause-4.lua → esm/scripts/pause-4.js} +8 -7
  140. package/dist/esm/scripts/pause-4.js.map +1 -0
  141. package/dist/esm/scripts/promote-6.d.ts +5 -0
  142. package/dist/esm/{commands/promote-6.lua → scripts/promote-6.js} +19 -14
  143. package/dist/esm/scripts/promote-6.js.map +1 -0
  144. package/dist/esm/scripts/releaseLock-1.d.ts +5 -0
  145. package/dist/{cjs/commands/releaseLock-1.lua → esm/scripts/releaseLock-1.js} +8 -5
  146. package/dist/esm/scripts/releaseLock-1.js.map +1 -0
  147. package/dist/esm/scripts/removeJob-1.d.ts +5 -0
  148. package/dist/esm/scripts/removeJob-1.js +203 -0
  149. package/dist/esm/scripts/removeJob-1.js.map +1 -0
  150. package/dist/esm/scripts/removeRepeatable-2.d.ts +5 -0
  151. package/dist/esm/{commands/removeRepeatable-2.lua → scripts/removeRepeatable-2.js} +8 -7
  152. package/dist/esm/scripts/removeRepeatable-2.js.map +1 -0
  153. package/dist/esm/scripts/reprocessJob-4.d.ts +5 -0
  154. package/dist/{cjs/commands/reprocessJob-4.lua → esm/scripts/reprocessJob-4.js} +8 -5
  155. package/dist/esm/scripts/reprocessJob-4.js.map +1 -0
  156. package/dist/esm/scripts/retryJob-6.d.ts +5 -0
  157. package/dist/{cjs/commands/retryJob-6.lua → esm/scripts/retryJob-6.js} +19 -13
  158. package/dist/esm/scripts/retryJob-6.js.map +1 -0
  159. package/dist/esm/scripts/retryJobs-6.d.ts +5 -0
  160. package/dist/esm/{commands/retryJobs-6.lua → scripts/retryJobs-6.js} +35 -15
  161. package/dist/esm/scripts/retryJobs-6.js.map +1 -0
  162. package/dist/esm/scripts/takeLock-1.d.ts +5 -0
  163. package/dist/esm/{commands/takeLock-1.lua → scripts/takeLock-1.js} +8 -4
  164. package/dist/esm/scripts/takeLock-1.js.map +1 -0
  165. package/dist/esm/scripts/updateData-1.d.ts +5 -0
  166. package/dist/esm/{commands/updateData-1.lua → scripts/updateData-1.js} +8 -4
  167. package/dist/esm/scripts/updateData-1.js.map +1 -0
  168. package/dist/esm/scripts/updateProgress-2.d.ts +5 -0
  169. package/dist/{cjs/commands/updateProgress-2.lua → esm/scripts/updateProgress-2.js} +8 -5
  170. package/dist/esm/scripts/updateProgress-2.js.map +1 -0
  171. package/package.json +4 -2
  172. package/dist/cjs/commands/cleanJobsInSet-2.lua +0 -46
  173. package/dist/cjs/commands/drain-4.lua +0 -25
  174. package/dist/cjs/commands/includes/addJobWithPriority.lua +0 -16
  175. package/dist/cjs/commands/includes/batches.lua +0 -18
  176. package/dist/cjs/commands/includes/checkItemInList.lua +0 -12
  177. package/dist/cjs/commands/includes/checkStalledJobs.lua +0 -137
  178. package/dist/cjs/commands/includes/cleanList.lua +0 -50
  179. package/dist/cjs/commands/includes/cleanSet.lua +0 -50
  180. package/dist/cjs/commands/includes/collectMetrics.lua +0 -46
  181. package/dist/cjs/commands/includes/destructureJobKey.lua +0 -12
  182. package/dist/cjs/commands/includes/getNextDelayedTimestamp.lua +0 -13
  183. package/dist/cjs/commands/includes/getTargetQueueList.lua +0 -12
  184. package/dist/cjs/commands/includes/getTimestamp.lua +0 -19
  185. package/dist/cjs/commands/includes/getZSetItems.lua +0 -7
  186. package/dist/cjs/commands/includes/isLocked.lua +0 -31
  187. package/dist/cjs/commands/includes/moveJobFromWaitToActive.lua +0 -84
  188. package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +0 -25
  189. package/dist/cjs/commands/includes/promoteDelayedJobs.lua +0 -49
  190. package/dist/cjs/commands/includes/removeJob.lua +0 -13
  191. package/dist/cjs/commands/includes/removeJobFromAnyState.lua +0 -26
  192. package/dist/cjs/commands/includes/removeJobs.lua +0 -38
  193. package/dist/cjs/commands/includes/removeJobsByMaxAge.lua +0 -15
  194. package/dist/cjs/commands/includes/removeJobsByMaxCount.lua +0 -15
  195. package/dist/cjs/commands/includes/trimEvents.lua +0 -12
  196. package/dist/cjs/commands/includes/updateParentDepsIfNeeded.lua +0 -28
  197. package/dist/cjs/commands/isJobInList-1.lua +0 -16
  198. package/dist/cjs/commands/moveStalledJobsToWait-8.lua +0 -24
  199. package/dist/cjs/commands/moveToActive-9.lua +0 -67
  200. package/dist/cjs/commands/moveToFinished-12.lua +0 -201
  201. package/dist/cjs/commands/obliterate-2.lua +0 -94
  202. package/dist/cjs/commands/removeJob-1.lua +0 -72
  203. package/dist/cjs/commands/updateData-1.lua +0 -16
  204. package/dist/esm/commands/cleanJobsInSet-2.lua +0 -46
  205. package/dist/esm/commands/drain-4.lua +0 -25
  206. package/dist/esm/commands/includes/addJobWithPriority.lua +0 -16
  207. package/dist/esm/commands/includes/batches.lua +0 -18
  208. package/dist/esm/commands/includes/checkItemInList.lua +0 -12
  209. package/dist/esm/commands/includes/checkStalledJobs.lua +0 -137
  210. package/dist/esm/commands/includes/cleanList.lua +0 -50
  211. package/dist/esm/commands/includes/cleanSet.lua +0 -50
  212. package/dist/esm/commands/includes/collectMetrics.lua +0 -46
  213. package/dist/esm/commands/includes/destructureJobKey.lua +0 -12
  214. package/dist/esm/commands/includes/getNextDelayedTimestamp.lua +0 -13
  215. package/dist/esm/commands/includes/getTargetQueueList.lua +0 -12
  216. package/dist/esm/commands/includes/getTimestamp.lua +0 -19
  217. package/dist/esm/commands/includes/getZSetItems.lua +0 -7
  218. package/dist/esm/commands/includes/isLocked.lua +0 -31
  219. package/dist/esm/commands/includes/moveJobFromWaitToActive.lua +0 -84
  220. package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +0 -25
  221. package/dist/esm/commands/includes/promoteDelayedJobs.lua +0 -49
  222. package/dist/esm/commands/includes/removeJob.lua +0 -13
  223. package/dist/esm/commands/includes/removeJobFromAnyState.lua +0 -26
  224. package/dist/esm/commands/includes/removeJobs.lua +0 -38
  225. package/dist/esm/commands/includes/removeJobsByMaxAge.lua +0 -15
  226. package/dist/esm/commands/includes/removeJobsByMaxCount.lua +0 -15
  227. package/dist/esm/commands/includes/trimEvents.lua +0 -12
  228. package/dist/esm/commands/includes/updateParentDepsIfNeeded.lua +0 -28
  229. package/dist/esm/commands/isJobInList-1.lua +0 -16
  230. package/dist/esm/commands/moveStalledJobsToWait-8.lua +0 -24
  231. package/dist/esm/commands/moveToActive-9.lua +0 -67
  232. package/dist/esm/commands/moveToFinished-12.lua +0 -201
  233. package/dist/esm/commands/obliterate-2.lua +0 -94
  234. package/dist/esm/commands/removeJob-1.lua +0 -72
@@ -0,0 +1,288 @@
1
+ const content = `--[[
2
+ Move stalled jobs to wait.
3
+ Input:
4
+ KEYS[1] 'stalled' (SET)
5
+ KEYS[2] 'wait', (LIST)
6
+ KEYS[3] 'active', (LIST)
7
+ KEYS[4] 'failed', (ZSET)
8
+ KEYS[5] 'stalled-check', (KEY)
9
+ KEYS[6] 'meta', (KEY)
10
+ KEYS[7] 'paused', (LIST)
11
+ KEYS[8] 'event stream' (STREAM)
12
+ ARGV[1] Max stalled job count
13
+ ARGV[2] queue.toKey('')
14
+ ARGV[3] timestamp
15
+ ARGV[4] max check time
16
+ Events:
17
+ 'stalled' with stalled job id.
18
+ ]] -- Includes
19
+ --[[
20
+ Move stalled jobs to wait.
21
+ Input:
22
+ stalledKey 'stalled' (SET)
23
+ waitKey 'wait', (LIST)
24
+ activeKey 'active', (LIST)
25
+ failedKey 'failed', (ZSET)
26
+ stalledCheckKey 'stalled-check', (KEY)
27
+ metaKey 'meta', (KEY)
28
+ pausedKey 'paused', (LIST)
29
+ eventStreamKey 'event stream' (STREAM)
30
+ maxStalledJobCount Max stalled job count
31
+ queueKeyPrefix queue.toKey('')
32
+ timestamp timestamp
33
+ maxCheckTime max check time
34
+ Events:
35
+ 'stalled' with stalled job id.
36
+ ]] local rcall = redis.call
37
+ -- Includes
38
+ --[[
39
+ Function to loop in batches.
40
+ Just a bit of warning, some commands as ZREM
41
+ could receive a maximum of 7000 parameters per call.
42
+ ]]
43
+ local function batches(n, batchSize)
44
+ local i = 0
45
+ return function()
46
+ local from = i * batchSize + 1
47
+ i = i + 1
48
+ if (from <= n) then
49
+ local to = math.min(from + batchSize - 1, n)
50
+ return from, to
51
+ end
52
+ end
53
+ end
54
+ --[[
55
+ Function to check for the meta.paused key to decide if we are paused or not
56
+ (since an empty list and !EXISTS are not really the same).
57
+ ]]
58
+ local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
59
+ if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
60
+ return waitKey
61
+ else
62
+ return pausedKey
63
+ end
64
+ end
65
+ --[[
66
+ Function to remove job.
67
+ ]]
68
+ -- Includes
69
+ --[[
70
+ Check if this job has a parent. If so we will just remove it from
71
+ the parent child list, but if it is the last child we should move the parent to "wait/paused"
72
+ which requires code from "moveToFinished"
73
+ ]]
74
+ --[[
75
+ Functions to destructure job key.
76
+ Just a bit of warning, these functions may be a bit slow and affect performance significantly.
77
+ ]]
78
+ local getJobIdFromKey = function (jobKey)
79
+ return string.match(jobKey, ".*:(.*)")
80
+ end
81
+ local getJobKeyPrefix = function (jobKey, jobId)
82
+ return string.sub(jobKey, 0, #jobKey - #jobId)
83
+ end
84
+ local function moveParentToWait(parentPrefix, parentId, emitEvent)
85
+ local parentTarget = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "wait", parentPrefix .. "paused")
86
+ rcall("RPUSH", parentTarget, parentId)
87
+ if emitEvent then
88
+ local parentEventStream = parentPrefix .. "events"
89
+ rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
90
+ end
91
+ end
92
+ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
93
+ if parentKey then
94
+ local parentProcessedKey = parentKey .. ":processed"
95
+ rcall("HDEL", parentProcessedKey, jobKey)
96
+ local parentDependenciesKey = parentKey .. ":dependencies"
97
+ local result = rcall("SREM", parentDependenciesKey, jobKey)
98
+ if result > 0 then
99
+ local pendingDependencies = rcall("SCARD", parentDependenciesKey)
100
+ if pendingDependencies == 0 then
101
+ local parentId = getJobIdFromKey(parentKey)
102
+ local parentPrefix = getJobKeyPrefix(parentKey, parentId)
103
+ local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
104
+ if numRemovedElements == 1 then
105
+ if hard then
106
+ if parentPrefix == baseKey then
107
+ removeParentDependencyKey(parentKey, hard, nil, baseKey)
108
+ rcall("DEL", parentKey, parentKey .. ':logs',
109
+ parentKey .. ':dependencies', parentKey .. ':processed')
110
+ else
111
+ moveParentToWait(parentPrefix, parentId)
112
+ end
113
+ else
114
+ moveParentToWait(parentPrefix, parentId, true)
115
+ end
116
+ end
117
+ end
118
+ end
119
+ else
120
+ local missedParentKey = rcall("HGET", jobKey, "parentKey")
121
+ if( (type(missedParentKey) == "string") and missedParentKey ~= "" and (rcall("EXISTS", missedParentKey) == 1)) then
122
+ local parentProcessedKey = missedParentKey .. ":processed"
123
+ rcall("HDEL", parentProcessedKey, jobKey)
124
+ local parentDependenciesKey = missedParentKey .. ":dependencies"
125
+ local result = rcall("SREM", parentDependenciesKey, jobKey)
126
+ if result > 0 then
127
+ local pendingDependencies = rcall("SCARD", parentDependenciesKey)
128
+ if pendingDependencies == 0 then
129
+ local parentId = getJobIdFromKey(missedParentKey)
130
+ local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
131
+ local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
132
+ if numRemovedElements == 1 then
133
+ if hard then
134
+ if parentPrefix == baseKey then
135
+ removeParentDependencyKey(missedParentKey, hard, nil, baseKey)
136
+ rcall("DEL", missedParentKey, missedParentKey .. ':logs',
137
+ missedParentKey .. ':dependencies', missedParentKey .. ':processed')
138
+ else
139
+ moveParentToWait(parentPrefix, parentId)
140
+ end
141
+ else
142
+ moveParentToWait(parentPrefix, parentId, true)
143
+ end
144
+ end
145
+ end
146
+ end
147
+ end
148
+ end
149
+ end
150
+ local function removeJob(jobId, hard, baseKey)
151
+ local jobKey = baseKey .. jobId
152
+ removeParentDependencyKey(jobKey, hard, nil, baseKey)
153
+ rcall("DEL", jobKey, jobKey .. ':logs',
154
+ jobKey .. ':dependencies', jobKey .. ':processed')
155
+ end
156
+ --[[
157
+ Functions to remove jobs by max age.
158
+ ]]
159
+ -- Includes
160
+ local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
161
+ local start = timestamp - maxAge * 1000
162
+ local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
163
+ for i, jobId in ipairs(jobIds) do
164
+ removeJob(jobId, false, prefix)
165
+ end
166
+ rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
167
+ end
168
+ --[[
169
+ Functions to remove jobs by max count.
170
+ ]]
171
+ -- Includes
172
+ local function removeJobsByMaxCount(maxCount, targetSet, prefix)
173
+ local start = maxCount
174
+ local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
175
+ for i, jobId in ipairs(jobIds) do
176
+ removeJob(jobId, false, prefix)
177
+ end
178
+ rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
179
+ end
180
+ --[[
181
+ Function to trim events, default 10000.
182
+ ]]
183
+ local function trimEvents(metaKey, eventStreamKey)
184
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
185
+ if maxEvents ~= false then
186
+ rcall("XTRIM", eventStreamKey, "MAXLEN", "~", maxEvents)
187
+ else
188
+ rcall("XTRIM", eventStreamKey, "MAXLEN", "~", 10000)
189
+ end
190
+ end
191
+ -- Check if we need to check for stalled jobs now.
192
+ local function checkStalledJobs(stalledKey, waitKey, activeKey, failedKey, stalledCheckKey,
193
+ metaKey, pausedKey, eventStreamKey, maxStalledJobCount, queueKeyPrefix, timestamp, maxCheckTime)
194
+ if rcall("EXISTS", stalledCheckKey) == 1 then
195
+ return {{}, {}}
196
+ end
197
+ rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
198
+ -- Trim events before emiting them to avoid trimming events emitted in this script
199
+ trimEvents(metaKey, eventStreamKey)
200
+ -- Move all stalled jobs to wait
201
+ local stalling = rcall('SMEMBERS', stalledKey)
202
+ local stalled = {}
203
+ local failed = {}
204
+ if (#stalling > 0) then
205
+ rcall('DEL', stalledKey)
206
+ local MAX_STALLED_JOB_COUNT = tonumber(maxStalledJobCount)
207
+ -- Remove from active list
208
+ for i, jobId in ipairs(stalling) do
209
+ if jobId == '0' then
210
+ -- If the jobId is a delay marker ID we just remove it.
211
+ local removed = rcall("LREM", activeKey, 1, jobId)
212
+ else
213
+ local jobKey = queueKeyPrefix .. jobId
214
+ -- Check that the lock is also missing, then we can handle this job as really stalled.
215
+ if (rcall("EXISTS", jobKey .. ":lock") == 0) then
216
+ -- Remove from the active queue.
217
+ local removed = rcall("LREM", activeKey, 1, jobId)
218
+ if (removed > 0) then
219
+ -- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
220
+ local stalledCount =
221
+ rcall("HINCRBY", jobKey, "stalledCounter", 1)
222
+ if (stalledCount > MAX_STALLED_JOB_COUNT) then
223
+ local rawOpts = rcall("HGET", jobKey, "opts")
224
+ local opts = cjson.decode(rawOpts)
225
+ local removeOnFailType = type(opts["removeOnFail"])
226
+ rcall("ZADD", failedKey, timestamp, jobId)
227
+ local failedReason =
228
+ "job stalled more than allowable limit"
229
+ rcall("HMSET", jobKey, "failedReason", failedReason,
230
+ "finishedOn", timestamp)
231
+ rcall("XADD", eventStreamKey, "*", "event", "failed", "jobId",
232
+ jobId, 'prev', 'active', 'failedReason',
233
+ failedReason)
234
+ if removeOnFailType == "number" then
235
+ removeJobsByMaxCount(opts["removeOnFail"], failedKey,
236
+ queueKeyPrefix)
237
+ elseif removeOnFailType == "boolean" then
238
+ if opts["removeOnFail"] then
239
+ removeJob(jobId, false, queueKeyPrefix)
240
+ rcall("ZREM", failedKey, jobId)
241
+ end
242
+ elseif removeOnFailType ~= "nil" then
243
+ local maxAge = opts["removeOnFail"]["age"]
244
+ local maxCount = opts["removeOnFail"]["count"]
245
+ if maxAge ~= nil then
246
+ removeJobsByMaxAge(timestamp, maxAge, failedKey,
247
+ queueKeyPrefix)
248
+ end
249
+ if maxCount ~= nil and maxCount > 0 then
250
+ removeJobsByMaxCount(maxCount, failedKey, queueKeyPrefix)
251
+ end
252
+ end
253
+ table.insert(failed, jobId)
254
+ else
255
+ local target = getTargetQueueList(metaKey, waitKey,
256
+ pausedKey)
257
+ -- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
258
+ rcall("RPUSH", target, jobId)
259
+ rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
260
+ jobId, 'prev', 'active')
261
+ -- Emit the stalled event
262
+ rcall("XADD", eventStreamKey, "*", "event", "stalled", "jobId",
263
+ jobId)
264
+ table.insert(stalled, jobId)
265
+ end
266
+ end
267
+ end
268
+ end
269
+ end
270
+ end
271
+ -- Mark potentially stalled jobs
272
+ local active = rcall('LRANGE', activeKey, 0, -1)
273
+ if (#active > 0) then
274
+ for from, to in batches(#active, 7000) do
275
+ rcall('SADD', stalledKey, unpack(active, from, to))
276
+ end
277
+ end
278
+ return {failed, stalled}
279
+ end
280
+ return checkStalledJobs(KEYS[1], KEYS[2], KEYS[3], KEYS[4], KEYS[5], KEYS[6],
281
+ KEYS[7], KEYS[8], ARGV[1], ARGV[2], ARGV[3], ARGV[4])
282
+ `;
283
+ export const moveStalledJobsToWait = {
284
+ name: 'moveStalledJobsToWait',
285
+ content,
286
+ keys: 8,
287
+ };
288
+ //# sourceMappingURL=moveStalledJobsToWait-8.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"moveStalledJobsToWait-8.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyRf,CAAC;AACF,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -0,0 +1,5 @@
1
+ export declare const moveToActive: {
2
+ name: string;
3
+ content: string;
4
+ keys: number;
5
+ };
@@ -0,0 +1,204 @@
1
+ const content = `--[[
2
+ Move next job to be processed to active, lock it and fetch its data. The job
3
+ may be delayed, in that case we need to move it to the delayed set instead.
4
+ This operation guarantees that the worker owns the job during the lock
5
+ expiration time. The worker is responsible of keeping the lock fresh
6
+ so that no other worker picks this job again.
7
+ Input:
8
+ KEYS[1] wait key
9
+ KEYS[2] active key
10
+ KEYS[3] priority key
11
+ KEYS[4] stream events key
12
+ KEYS[5] stalled key
13
+ -- Rate limiting
14
+ KEYS[6] rate limiter key
15
+ KEYS[7] delayed key
16
+ -- Promote delayed jobs
17
+ KEYS[8] paused key
18
+ KEYS[9] meta key
19
+ -- Arguments
20
+ ARGV[1] key prefix
21
+ ARGV[2] timestamp
22
+ ARGV[3] optional job ID
23
+ ARGV[4] opts
24
+ opts - token - lock token
25
+ opts - lockDuration
26
+ opts - limiter
27
+ ]]
28
+ local jobId
29
+ local rcall = redis.call
30
+ -- Includes
31
+ --[[
32
+ Function to move job from wait state to active.
33
+ Input:
34
+ keys[1] wait key
35
+ keys[2] active key
36
+ keys[3] priority key
37
+ keys[4] stream events key
38
+ keys[5] stalled key
39
+ -- Rate limiting
40
+ keys[6] rate limiter key
41
+ keys[7] delayed key
42
+ opts - token - lock token
43
+ opts - lockDuration
44
+ opts - limiter
45
+ ]]
46
+ local function moveJobFromWaitToActive(keys, keyPrefix, jobId, processedOn, opts)
47
+ -- Check if we need to perform rate limiting.
48
+ local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
49
+ if(maxJobs) then
50
+ local rateLimiterKey = keys[6];
51
+ local groupKey
52
+ local groupKeyOpt = opts['limiter'] and opts['limiter']['groupKey'] or ""
53
+ if groupKeyOpt ~= "" then
54
+ groupKey = string.match(jobId, "[^:]+$")
55
+ if groupKey ~= jobId then
56
+ rateLimiterKey = rateLimiterKey .. ":" .. groupKey
57
+ end
58
+ end
59
+ local jobCounter
60
+ if groupKey ~= nil then
61
+ if rateLimiterKey ~= keys[6] then
62
+ jobCounter = tonumber(rcall("INCR", rateLimiterKey))
63
+ end
64
+ else
65
+ jobCounter = tonumber(rcall("INCR", rateLimiterKey))
66
+ end
67
+ local limiterDuration = opts['limiter'] and opts['limiter']['duration']
68
+ -- check if rate limit hit
69
+ if jobCounter ~= nil and jobCounter > maxJobs then
70
+ local exceedingJobs = jobCounter - maxJobs
71
+ local expireTime = tonumber(rcall("PTTL", rateLimiterKey))
72
+ local delay = expireTime + ((exceedingJobs - 1) * limiterDuration) / maxJobs;
73
+ local timestamp = delay + tonumber(processedOn)
74
+ -- put job into delayed queue
75
+ rcall("ZADD", keys[7], timestamp * 0x1000 + bit.band(jobCounter, 0xfff), jobId);
76
+ rcall("XADD", keys[4], "*", "event", "delayed", "jobId", jobId, "delay", timestamp);
77
+ -- remove from active queue
78
+ rcall("LREM", keys[2], 1, jobId)
79
+ -- Return when we can process more jobs
80
+ return expireTime
81
+ else
82
+ if jobCounter == 1 then
83
+ rcall("PEXPIRE", rateLimiterKey, limiterDuration)
84
+ end
85
+ end
86
+ end
87
+ local jobKey = keyPrefix .. jobId
88
+ local lockKey = jobKey .. ':lock'
89
+ -- get a lock
90
+ if opts['token'] ~= "0" then
91
+ rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
92
+ end
93
+ rcall("ZREM", keys[3], jobId) -- remove from priority
94
+ rcall("XADD", keys[4], "*", "event", "active", "jobId", jobId, "prev", "waiting")
95
+ rcall("HSET", jobKey, "processedOn", processedOn)
96
+ rcall("HINCRBY", jobKey, "attemptsMade", 1)
97
+ return {rcall("HGETALL", jobKey), jobId} -- get job data
98
+ end
99
+ --[[
100
+ Function to return the next delayed job timestamp.
101
+ ]]
102
+ local function getNextDelayedTimestamp(delayedKey)
103
+ local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
104
+ if #result then
105
+ local nextTimestamp = tonumber(result[2])
106
+ if (nextTimestamp ~= nil) then
107
+ nextTimestamp = nextTimestamp / 0x1000
108
+ end
109
+ return nextTimestamp
110
+ end
111
+ end
112
+ --[[
113
+ Updates the delay set, by moving delayed jobs that should
114
+ be processed now to "wait".
115
+ Events:
116
+ 'waiting'
117
+ ]]
118
+ local rcall = redis.call
119
+ -- Includes
120
+ --[[
121
+ Function to add job considering priority.
122
+ ]]
123
+ local function addJobWithPriority(priorityKey, priority, targetKey, jobId)
124
+ rcall("ZADD", priorityKey, priority, jobId)
125
+ local count = rcall("ZCOUNT", priorityKey, 0, priority)
126
+ local len = rcall("LLEN", targetKey)
127
+ local id = rcall("LINDEX", targetKey, len - (count - 1))
128
+ if id then
129
+ rcall("LINSERT", targetKey, "BEFORE", id, jobId)
130
+ else
131
+ rcall("RPUSH", targetKey, jobId)
132
+ end
133
+ end
134
+ --[[
135
+ Function to check for the meta.paused key to decide if we are paused or not
136
+ (since an empty list and !EXISTS are not really the same).
137
+ ]]
138
+ local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
139
+ if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
140
+ return waitKey
141
+ else
142
+ return pausedKey
143
+ end
144
+ end
145
+ -- Try to get as much as 1000 jobs at once, and returns the nextTimestamp if
146
+ -- there are more delayed jobs to process.
147
+ local function promoteDelayedJobs(delayedKey, waitKey, priorityKey, pausedKey,
148
+ metaKey, eventStreamKey, prefix, timestamp)
149
+ local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000, "LIMIT", 0, 1000)
150
+ if (#jobs > 0) then
151
+ rcall("ZREM", delayedKey, unpack(jobs))
152
+ -- check if we need to use push in paused instead of waiting
153
+ local target = getTargetQueueList(metaKey, waitKey, pausedKey)
154
+ for _, jobId in ipairs(jobs) do
155
+ local priority =
156
+ tonumber(rcall("HGET", prefix .. jobId, "priority")) or 0
157
+ if priority == 0 then
158
+ -- LIFO or FIFO
159
+ rcall("LPUSH", target, jobId)
160
+ else
161
+ addJobWithPriority(priorityKey, priority, target, jobId)
162
+ end
163
+ -- Emit waiting event
164
+ rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
165
+ jobId, "prev", "delayed")
166
+ rcall("HSET", prefix .. jobId, "delay", 0)
167
+ end
168
+ end
169
+ local nextTimestamp = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")[2]
170
+ if (nextTimestamp ~= nil) then
171
+ nextTimestamp = nextTimestamp / 0x1000
172
+ end
173
+ return nextTimestamp
174
+ end
175
+ -- Check if there are delayed jobs that we can move to wait.
176
+ promoteDelayedJobs(KEYS[7], KEYS[1], KEYS[3], KEYS[8], KEYS[9], KEYS[4], ARGV[1], ARGV[2])
177
+ if (ARGV[3] ~= "") then
178
+ jobId = ARGV[3]
179
+ -- clean stalled key
180
+ rcall("SREM", KEYS[5], jobId)
181
+ else
182
+ -- no job ID, try non-blocking move from wait to active
183
+ jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
184
+ end
185
+ -- If jobId is special ID 0, then there is no job to process
186
+ if jobId == "0" then
187
+ rcall("LREM", KEYS[2], 1, 0)
188
+ elseif jobId then
189
+ local opts = cmsgpack.unpack(ARGV[4])
190
+ -- this script is not really moving, it is preparing the job for processing
191
+ return moveJobFromWaitToActive(KEYS, ARGV[1], jobId, ARGV[2], opts)
192
+ end
193
+ -- Return the timestamp for the next delayed job if any.
194
+ local nextTimestamp = getNextDelayedTimestamp(KEYS[7])
195
+ if (nextTimestamp ~= nil) then
196
+ return nextTimestamp - tonumber(ARGV[2])
197
+ end
198
+ `;
199
+ export const moveToActive = {
200
+ name: 'moveToActive',
201
+ content,
202
+ keys: 9,
203
+ };
204
+ //# sourceMappingURL=moveToActive-9.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"moveToActive-9.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-9.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqMf,CAAC;AACF,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -0,0 +1,5 @@
1
+ export declare const moveToDelayed: {
2
+ name: string;
3
+ content: string;
4
+ keys: number;
5
+ };
@@ -1,29 +1,23 @@
1
- --[[
1
+ const content = `--[[
2
2
  Moves job from active to delayed set.
3
-
4
3
  Input:
5
4
  KEYS[1] active key
6
5
  KEYS[2] delayed key
7
6
  KEYS[3] job key
8
7
  KEYS[4] events stream
9
8
  KEYS[5] delayed stream
10
-
11
9
  ARGV[1] delayedTimestamp
12
10
  ARGV[2] the id of the job
13
11
  ARGV[3] queue token
14
-
15
12
  Output:
16
13
  0 - OK
17
14
  -1 - Missing job.
18
15
  -3 - Job not in active set.
19
-
20
16
  Events:
21
17
  - delayed key.
22
18
  ]]
23
19
  local rcall = redis.call
24
-
25
20
  if rcall("EXISTS", KEYS[3]) == 1 then
26
-
27
21
  if ARGV[3] ~= "0" then
28
22
  local lockKey = KEYS[3] .. ':lock'
29
23
  if rcall("GET", lockKey) == ARGV[3] then
@@ -32,23 +26,24 @@ if rcall("EXISTS", KEYS[3]) == 1 then
32
26
  return -2
33
27
  end
34
28
  end
35
-
36
29
  local jobId = ARGV[2]
37
30
  local score = tonumber(ARGV[1])
38
31
  local delayedTimestamp = (score / 0x1000)
39
-
40
32
  local numRemovedElements = rcall("LREM", KEYS[1], -1, jobId)
41
-
42
33
  if(numRemovedElements < 1) then
43
34
  return -3
44
35
  end
45
-
46
36
  rcall("ZADD", KEYS[2], score, jobId)
47
-
48
37
  rcall("XADD", KEYS[4], "*", "event", "delayed", "jobId", jobId, "delay", delayedTimestamp);
49
38
  rcall("XADD", KEYS[5], "*", "nextTimestamp", delayedTimestamp);
50
-
51
39
  return 0
52
40
  else
53
41
  return -1
54
42
  end
43
+ `;
44
+ export const moveToDelayed = {
45
+ name: 'moveToDelayed',
46
+ content,
47
+ keys: 5,
48
+ };
49
+ //# sourceMappingURL=moveToDelayed-5.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"moveToDelayed-5.js","sourceRoot":"","sources":["../../../src/scripts/moveToDelayed-5.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0Cf,CAAC;AACF,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -0,0 +1,5 @@
1
+ export declare const moveToFinished: {
2
+ name: string;
3
+ content: string;
4
+ keys: number;
5
+ };