bullmq 2.2.0 → 2.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (234) hide show
  1. package/dist/cjs/classes/job.d.ts +1 -1
  2. package/dist/cjs/classes/redis-connection.d.ts +6 -2
  3. package/dist/cjs/classes/redis-connection.js +11 -5
  4. package/dist/cjs/classes/redis-connection.js.map +1 -1
  5. package/dist/cjs/scripts/addJob-8.d.ts +5 -0
  6. package/dist/{esm/commands/addJob-8.lua → cjs/scripts/addJob-8.js} +83 -31
  7. package/dist/cjs/scripts/addJob-8.js.map +1 -0
  8. package/dist/cjs/scripts/changeDelay-4.d.ts +5 -0
  9. package/dist/cjs/{commands/changeDelay-4.lua → scripts/changeDelay-4.js} +11 -11
  10. package/dist/cjs/scripts/changeDelay-4.js.map +1 -0
  11. package/dist/cjs/scripts/cleanJobsInSet-2.d.ts +5 -0
  12. package/dist/cjs/scripts/cleanJobsInSet-2.js +264 -0
  13. package/dist/cjs/scripts/cleanJobsInSet-2.js.map +1 -0
  14. package/dist/cjs/scripts/drain-4.d.ts +5 -0
  15. package/dist/cjs/{commands/includes/removeParentDependencyKey.lua → scripts/drain-4.js} +110 -10
  16. package/dist/cjs/scripts/drain-4.js.map +1 -0
  17. package/dist/cjs/scripts/extendLock-2.d.ts +5 -0
  18. package/dist/{esm/commands/extendLock-2.lua → cjs/scripts/extendLock-2.js} +11 -4
  19. package/dist/cjs/scripts/extendLock-2.js.map +1 -0
  20. package/dist/cjs/scripts/getState-7.d.ts +5 -0
  21. package/dist/{esm/commands/getState-7.lua → cjs/scripts/getState-7.js} +22 -12
  22. package/dist/cjs/scripts/getState-7.js.map +1 -0
  23. package/dist/cjs/scripts/getStateV2-7.d.ts +5 -0
  24. package/dist/cjs/{commands/getStateV2-7.lua → scripts/getStateV2-7.js} +11 -10
  25. package/dist/cjs/scripts/getStateV2-7.js.map +1 -0
  26. package/dist/cjs/scripts/index.d.ts +26 -0
  27. package/dist/cjs/scripts/index.js +30 -0
  28. package/dist/cjs/scripts/index.js.map +1 -0
  29. package/dist/cjs/scripts/isFinished-3.d.ts +5 -0
  30. package/dist/cjs/{commands/isFinished-3.lua → scripts/isFinished-3.js} +11 -10
  31. package/dist/cjs/scripts/isFinished-3.js.map +1 -0
  32. package/dist/cjs/scripts/isJobInList-1.d.ts +5 -0
  33. package/dist/cjs/scripts/isJobInList-1.js +32 -0
  34. package/dist/cjs/scripts/isJobInList-1.js.map +1 -0
  35. package/dist/cjs/scripts/moveStalledJobsToWait-8.d.ts +5 -0
  36. package/dist/cjs/scripts/moveStalledJobsToWait-8.js +291 -0
  37. package/dist/cjs/scripts/moveStalledJobsToWait-8.js.map +1 -0
  38. package/dist/cjs/scripts/moveToActive-9.d.ts +5 -0
  39. package/dist/cjs/scripts/moveToActive-9.js +207 -0
  40. package/dist/cjs/scripts/moveToActive-9.js.map +1 -0
  41. package/dist/cjs/scripts/moveToDelayed-5.d.ts +5 -0
  42. package/dist/cjs/{commands/moveToDelayed-5.lua → scripts/moveToDelayed-5.js} +11 -13
  43. package/dist/cjs/scripts/moveToDelayed-5.js.map +1 -0
  44. package/dist/cjs/scripts/moveToFinished-12.d.ts +5 -0
  45. package/dist/cjs/scripts/moveToFinished-12.js +538 -0
  46. package/dist/cjs/scripts/moveToFinished-12.js.map +1 -0
  47. package/dist/cjs/scripts/moveToWaitingChildren-4.d.ts +5 -0
  48. package/dist/cjs/{commands/moveToWaitingChildren-4.lua → scripts/moveToWaitingChildren-4.js} +11 -14
  49. package/dist/cjs/scripts/moveToWaitingChildren-4.js.map +1 -0
  50. package/dist/cjs/scripts/obliterate-2.d.ts +5 -0
  51. package/dist/cjs/scripts/obliterate-2.js +241 -0
  52. package/dist/cjs/scripts/obliterate-2.js.map +1 -0
  53. package/dist/cjs/scripts/pause-4.d.ts +5 -0
  54. package/dist/{esm/commands/pause-4.lua → cjs/scripts/pause-4.js} +11 -7
  55. package/dist/cjs/scripts/pause-4.js.map +1 -0
  56. package/dist/cjs/scripts/promote-6.d.ts +5 -0
  57. package/dist/cjs/{commands/promote-6.lua → scripts/promote-6.js} +22 -14
  58. package/dist/cjs/scripts/promote-6.js.map +1 -0
  59. package/dist/cjs/scripts/releaseLock-1.d.ts +5 -0
  60. package/dist/{esm/commands/releaseLock-1.lua → cjs/scripts/releaseLock-1.js} +11 -5
  61. package/dist/cjs/scripts/releaseLock-1.js.map +1 -0
  62. package/dist/cjs/scripts/removeJob-1.d.ts +5 -0
  63. package/dist/cjs/scripts/removeJob-1.js +206 -0
  64. package/dist/cjs/scripts/removeJob-1.js.map +1 -0
  65. package/dist/cjs/scripts/removeRepeatable-2.d.ts +5 -0
  66. package/dist/cjs/{commands/removeRepeatable-2.lua → scripts/removeRepeatable-2.js} +11 -7
  67. package/dist/cjs/scripts/removeRepeatable-2.js.map +1 -0
  68. package/dist/cjs/scripts/reprocessJob-4.d.ts +5 -0
  69. package/dist/{esm/commands/reprocessJob-4.lua → cjs/scripts/reprocessJob-4.js} +11 -5
  70. package/dist/cjs/scripts/reprocessJob-4.js.map +1 -0
  71. package/dist/cjs/scripts/retryJob-6.d.ts +5 -0
  72. package/dist/{esm/commands/retryJob-6.lua → cjs/scripts/retryJob-6.js} +22 -13
  73. package/dist/cjs/scripts/retryJob-6.js.map +1 -0
  74. package/dist/cjs/scripts/retryJobs-6.d.ts +5 -0
  75. package/dist/cjs/{commands/retryJobs-6.lua → scripts/retryJobs-6.js} +38 -15
  76. package/dist/cjs/scripts/retryJobs-6.js.map +1 -0
  77. package/dist/cjs/scripts/takeLock-1.d.ts +5 -0
  78. package/dist/cjs/{commands/takeLock-1.lua → scripts/takeLock-1.js} +11 -4
  79. package/dist/cjs/scripts/takeLock-1.js.map +1 -0
  80. package/dist/cjs/scripts/updateData-1.d.ts +5 -0
  81. package/dist/cjs/scripts/updateData-1.js +23 -0
  82. package/dist/cjs/scripts/updateData-1.js.map +1 -0
  83. package/dist/cjs/scripts/updateProgress-2.d.ts +5 -0
  84. package/dist/{esm/commands/updateProgress-2.lua → cjs/scripts/updateProgress-2.js} +11 -5
  85. package/dist/cjs/scripts/updateProgress-2.js.map +1 -0
  86. package/dist/esm/classes/job.d.ts +1 -1
  87. package/dist/esm/classes/redis-connection.d.ts +6 -2
  88. package/dist/esm/classes/redis-connection.js +11 -5
  89. package/dist/esm/classes/redis-connection.js.map +1 -1
  90. package/dist/esm/scripts/addJob-8.d.ts +5 -0
  91. package/dist/{cjs/commands/addJob-8.lua → esm/scripts/addJob-8.js} +80 -31
  92. package/dist/esm/scripts/addJob-8.js.map +1 -0
  93. package/dist/esm/scripts/changeDelay-4.d.ts +5 -0
  94. package/dist/esm/{commands/changeDelay-4.lua → scripts/changeDelay-4.js} +8 -11
  95. package/dist/esm/scripts/changeDelay-4.js.map +1 -0
  96. package/dist/esm/scripts/cleanJobsInSet-2.d.ts +5 -0
  97. package/dist/esm/scripts/cleanJobsInSet-2.js +261 -0
  98. package/dist/esm/scripts/cleanJobsInSet-2.js.map +1 -0
  99. package/dist/esm/scripts/drain-4.d.ts +5 -0
  100. package/dist/esm/{commands/includes/removeParentDependencyKey.lua → scripts/drain-4.js} +107 -10
  101. package/dist/esm/scripts/drain-4.js.map +1 -0
  102. package/dist/esm/scripts/extendLock-2.d.ts +5 -0
  103. package/dist/{cjs/commands/extendLock-2.lua → esm/scripts/extendLock-2.js} +8 -4
  104. package/dist/esm/scripts/extendLock-2.js.map +1 -0
  105. package/dist/esm/scripts/getState-7.d.ts +5 -0
  106. package/dist/{cjs/commands/getState-7.lua → esm/scripts/getState-7.js} +19 -12
  107. package/dist/esm/scripts/getState-7.js.map +1 -0
  108. package/dist/esm/scripts/getStateV2-7.d.ts +5 -0
  109. package/dist/esm/{commands/getStateV2-7.lua → scripts/getStateV2-7.js} +8 -10
  110. package/dist/esm/scripts/getStateV2-7.js.map +1 -0
  111. package/dist/esm/scripts/index.d.ts +26 -0
  112. package/dist/esm/scripts/index.js +27 -0
  113. package/dist/esm/scripts/index.js.map +1 -0
  114. package/dist/esm/scripts/isFinished-3.d.ts +5 -0
  115. package/dist/esm/{commands/isFinished-3.lua → scripts/isFinished-3.js} +8 -10
  116. package/dist/esm/scripts/isFinished-3.js.map +1 -0
  117. package/dist/esm/scripts/isJobInList-1.d.ts +5 -0
  118. package/dist/esm/scripts/isJobInList-1.js +29 -0
  119. package/dist/esm/scripts/isJobInList-1.js.map +1 -0
  120. package/dist/esm/scripts/moveStalledJobsToWait-8.d.ts +5 -0
  121. package/dist/esm/scripts/moveStalledJobsToWait-8.js +288 -0
  122. package/dist/esm/scripts/moveStalledJobsToWait-8.js.map +1 -0
  123. package/dist/esm/scripts/moveToActive-9.d.ts +5 -0
  124. package/dist/esm/scripts/moveToActive-9.js +204 -0
  125. package/dist/esm/scripts/moveToActive-9.js.map +1 -0
  126. package/dist/esm/scripts/moveToDelayed-5.d.ts +5 -0
  127. package/dist/esm/{commands/moveToDelayed-5.lua → scripts/moveToDelayed-5.js} +8 -13
  128. package/dist/esm/scripts/moveToDelayed-5.js.map +1 -0
  129. package/dist/esm/scripts/moveToFinished-12.d.ts +5 -0
  130. package/dist/esm/scripts/moveToFinished-12.js +535 -0
  131. package/dist/esm/scripts/moveToFinished-12.js.map +1 -0
  132. package/dist/esm/scripts/moveToWaitingChildren-4.d.ts +5 -0
  133. package/dist/esm/{commands/moveToWaitingChildren-4.lua → scripts/moveToWaitingChildren-4.js} +8 -14
  134. package/dist/esm/scripts/moveToWaitingChildren-4.js.map +1 -0
  135. package/dist/esm/scripts/obliterate-2.d.ts +5 -0
  136. package/dist/esm/scripts/obliterate-2.js +238 -0
  137. package/dist/esm/scripts/obliterate-2.js.map +1 -0
  138. package/dist/esm/scripts/pause-4.d.ts +5 -0
  139. package/dist/{cjs/commands/pause-4.lua → esm/scripts/pause-4.js} +8 -7
  140. package/dist/esm/scripts/pause-4.js.map +1 -0
  141. package/dist/esm/scripts/promote-6.d.ts +5 -0
  142. package/dist/esm/{commands/promote-6.lua → scripts/promote-6.js} +19 -14
  143. package/dist/esm/scripts/promote-6.js.map +1 -0
  144. package/dist/esm/scripts/releaseLock-1.d.ts +5 -0
  145. package/dist/{cjs/commands/releaseLock-1.lua → esm/scripts/releaseLock-1.js} +8 -5
  146. package/dist/esm/scripts/releaseLock-1.js.map +1 -0
  147. package/dist/esm/scripts/removeJob-1.d.ts +5 -0
  148. package/dist/esm/scripts/removeJob-1.js +203 -0
  149. package/dist/esm/scripts/removeJob-1.js.map +1 -0
  150. package/dist/esm/scripts/removeRepeatable-2.d.ts +5 -0
  151. package/dist/esm/{commands/removeRepeatable-2.lua → scripts/removeRepeatable-2.js} +8 -7
  152. package/dist/esm/scripts/removeRepeatable-2.js.map +1 -0
  153. package/dist/esm/scripts/reprocessJob-4.d.ts +5 -0
  154. package/dist/{cjs/commands/reprocessJob-4.lua → esm/scripts/reprocessJob-4.js} +8 -5
  155. package/dist/esm/scripts/reprocessJob-4.js.map +1 -0
  156. package/dist/esm/scripts/retryJob-6.d.ts +5 -0
  157. package/dist/{cjs/commands/retryJob-6.lua → esm/scripts/retryJob-6.js} +19 -13
  158. package/dist/esm/scripts/retryJob-6.js.map +1 -0
  159. package/dist/esm/scripts/retryJobs-6.d.ts +5 -0
  160. package/dist/esm/{commands/retryJobs-6.lua → scripts/retryJobs-6.js} +35 -15
  161. package/dist/esm/scripts/retryJobs-6.js.map +1 -0
  162. package/dist/esm/scripts/takeLock-1.d.ts +5 -0
  163. package/dist/esm/{commands/takeLock-1.lua → scripts/takeLock-1.js} +8 -4
  164. package/dist/esm/scripts/takeLock-1.js.map +1 -0
  165. package/dist/esm/scripts/updateData-1.d.ts +5 -0
  166. package/dist/esm/{commands/updateData-1.lua → scripts/updateData-1.js} +8 -4
  167. package/dist/esm/scripts/updateData-1.js.map +1 -0
  168. package/dist/esm/scripts/updateProgress-2.d.ts +5 -0
  169. package/dist/{cjs/commands/updateProgress-2.lua → esm/scripts/updateProgress-2.js} +8 -5
  170. package/dist/esm/scripts/updateProgress-2.js.map +1 -0
  171. package/package.json +4 -2
  172. package/dist/cjs/commands/cleanJobsInSet-2.lua +0 -46
  173. package/dist/cjs/commands/drain-4.lua +0 -25
  174. package/dist/cjs/commands/includes/addJobWithPriority.lua +0 -16
  175. package/dist/cjs/commands/includes/batches.lua +0 -18
  176. package/dist/cjs/commands/includes/checkItemInList.lua +0 -12
  177. package/dist/cjs/commands/includes/checkStalledJobs.lua +0 -137
  178. package/dist/cjs/commands/includes/cleanList.lua +0 -50
  179. package/dist/cjs/commands/includes/cleanSet.lua +0 -50
  180. package/dist/cjs/commands/includes/collectMetrics.lua +0 -46
  181. package/dist/cjs/commands/includes/destructureJobKey.lua +0 -12
  182. package/dist/cjs/commands/includes/getNextDelayedTimestamp.lua +0 -13
  183. package/dist/cjs/commands/includes/getTargetQueueList.lua +0 -12
  184. package/dist/cjs/commands/includes/getTimestamp.lua +0 -19
  185. package/dist/cjs/commands/includes/getZSetItems.lua +0 -7
  186. package/dist/cjs/commands/includes/isLocked.lua +0 -31
  187. package/dist/cjs/commands/includes/moveJobFromWaitToActive.lua +0 -84
  188. package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +0 -25
  189. package/dist/cjs/commands/includes/promoteDelayedJobs.lua +0 -49
  190. package/dist/cjs/commands/includes/removeJob.lua +0 -13
  191. package/dist/cjs/commands/includes/removeJobFromAnyState.lua +0 -26
  192. package/dist/cjs/commands/includes/removeJobs.lua +0 -38
  193. package/dist/cjs/commands/includes/removeJobsByMaxAge.lua +0 -15
  194. package/dist/cjs/commands/includes/removeJobsByMaxCount.lua +0 -15
  195. package/dist/cjs/commands/includes/trimEvents.lua +0 -12
  196. package/dist/cjs/commands/includes/updateParentDepsIfNeeded.lua +0 -28
  197. package/dist/cjs/commands/isJobInList-1.lua +0 -16
  198. package/dist/cjs/commands/moveStalledJobsToWait-8.lua +0 -24
  199. package/dist/cjs/commands/moveToActive-9.lua +0 -67
  200. package/dist/cjs/commands/moveToFinished-12.lua +0 -201
  201. package/dist/cjs/commands/obliterate-2.lua +0 -94
  202. package/dist/cjs/commands/removeJob-1.lua +0 -72
  203. package/dist/cjs/commands/updateData-1.lua +0 -16
  204. package/dist/esm/commands/cleanJobsInSet-2.lua +0 -46
  205. package/dist/esm/commands/drain-4.lua +0 -25
  206. package/dist/esm/commands/includes/addJobWithPriority.lua +0 -16
  207. package/dist/esm/commands/includes/batches.lua +0 -18
  208. package/dist/esm/commands/includes/checkItemInList.lua +0 -12
  209. package/dist/esm/commands/includes/checkStalledJobs.lua +0 -137
  210. package/dist/esm/commands/includes/cleanList.lua +0 -50
  211. package/dist/esm/commands/includes/cleanSet.lua +0 -50
  212. package/dist/esm/commands/includes/collectMetrics.lua +0 -46
  213. package/dist/esm/commands/includes/destructureJobKey.lua +0 -12
  214. package/dist/esm/commands/includes/getNextDelayedTimestamp.lua +0 -13
  215. package/dist/esm/commands/includes/getTargetQueueList.lua +0 -12
  216. package/dist/esm/commands/includes/getTimestamp.lua +0 -19
  217. package/dist/esm/commands/includes/getZSetItems.lua +0 -7
  218. package/dist/esm/commands/includes/isLocked.lua +0 -31
  219. package/dist/esm/commands/includes/moveJobFromWaitToActive.lua +0 -84
  220. package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +0 -25
  221. package/dist/esm/commands/includes/promoteDelayedJobs.lua +0 -49
  222. package/dist/esm/commands/includes/removeJob.lua +0 -13
  223. package/dist/esm/commands/includes/removeJobFromAnyState.lua +0 -26
  224. package/dist/esm/commands/includes/removeJobs.lua +0 -38
  225. package/dist/esm/commands/includes/removeJobsByMaxAge.lua +0 -15
  226. package/dist/esm/commands/includes/removeJobsByMaxCount.lua +0 -15
  227. package/dist/esm/commands/includes/trimEvents.lua +0 -12
  228. package/dist/esm/commands/includes/updateParentDepsIfNeeded.lua +0 -28
  229. package/dist/esm/commands/isJobInList-1.lua +0 -16
  230. package/dist/esm/commands/moveStalledJobsToWait-8.lua +0 -24
  231. package/dist/esm/commands/moveToActive-9.lua +0 -67
  232. package/dist/esm/commands/moveToFinished-12.lua +0 -201
  233. package/dist/esm/commands/obliterate-2.lua +0 -94
  234. package/dist/esm/commands/removeJob-1.lua +0 -72
@@ -0,0 +1,291 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.moveStalledJobsToWait = void 0;
4
+ const content = `--[[
5
+ Move stalled jobs to wait.
6
+ Input:
7
+ KEYS[1] 'stalled' (SET)
8
+ KEYS[2] 'wait', (LIST)
9
+ KEYS[3] 'active', (LIST)
10
+ KEYS[4] 'failed', (ZSET)
11
+ KEYS[5] 'stalled-check', (KEY)
12
+ KEYS[6] 'meta', (KEY)
13
+ KEYS[7] 'paused', (LIST)
14
+ KEYS[8] 'event stream' (STREAM)
15
+ ARGV[1] Max stalled job count
16
+ ARGV[2] queue.toKey('')
17
+ ARGV[3] timestamp
18
+ ARGV[4] max check time
19
+ Events:
20
+ 'stalled' with stalled job id.
21
+ ]] -- Includes
22
+ --[[
23
+ Move stalled jobs to wait.
24
+ Input:
25
+ stalledKey 'stalled' (SET)
26
+ waitKey 'wait', (LIST)
27
+ activeKey 'active', (LIST)
28
+ failedKey 'failed', (ZSET)
29
+ stalledCheckKey 'stalled-check', (KEY)
30
+ metaKey 'meta', (KEY)
31
+ pausedKey 'paused', (LIST)
32
+ eventStreamKey 'event stream' (STREAM)
33
+ maxStalledJobCount Max stalled job count
34
+ queueKeyPrefix queue.toKey('')
35
+ timestamp timestamp
36
+ maxCheckTime max check time
37
+ Events:
38
+ 'stalled' with stalled job id.
39
+ ]] local rcall = redis.call
40
+ -- Includes
41
+ --[[
42
+ Function to loop in batches.
43
+ Just a bit of warning, some commands as ZREM
44
+ could receive a maximum of 7000 parameters per call.
45
+ ]]
46
+ local function batches(n, batchSize)
47
+ local i = 0
48
+ return function()
49
+ local from = i * batchSize + 1
50
+ i = i + 1
51
+ if (from <= n) then
52
+ local to = math.min(from + batchSize - 1, n)
53
+ return from, to
54
+ end
55
+ end
56
+ end
57
+ --[[
58
+ Function to check for the meta.paused key to decide if we are paused or not
59
+ (since an empty list and !EXISTS are not really the same).
60
+ ]]
61
+ local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
62
+ if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
63
+ return waitKey
64
+ else
65
+ return pausedKey
66
+ end
67
+ end
68
+ --[[
69
+ Function to remove job.
70
+ ]]
71
+ -- Includes
72
+ --[[
73
+ Check if this job has a parent. If so we will just remove it from
74
+ the parent child list, but if it is the last child we should move the parent to "wait/paused"
75
+ which requires code from "moveToFinished"
76
+ ]]
77
+ --[[
78
+ Functions to destructure job key.
79
+ Just a bit of warning, these functions may be a bit slow and affect performance significantly.
80
+ ]]
81
+ local getJobIdFromKey = function (jobKey)
82
+ return string.match(jobKey, ".*:(.*)")
83
+ end
84
+ local getJobKeyPrefix = function (jobKey, jobId)
85
+ return string.sub(jobKey, 0, #jobKey - #jobId)
86
+ end
87
+ local function moveParentToWait(parentPrefix, parentId, emitEvent)
88
+ local parentTarget = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "wait", parentPrefix .. "paused")
89
+ rcall("RPUSH", parentTarget, parentId)
90
+ if emitEvent then
91
+ local parentEventStream = parentPrefix .. "events"
92
+ rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
93
+ end
94
+ end
95
+ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
96
+ if parentKey then
97
+ local parentProcessedKey = parentKey .. ":processed"
98
+ rcall("HDEL", parentProcessedKey, jobKey)
99
+ local parentDependenciesKey = parentKey .. ":dependencies"
100
+ local result = rcall("SREM", parentDependenciesKey, jobKey)
101
+ if result > 0 then
102
+ local pendingDependencies = rcall("SCARD", parentDependenciesKey)
103
+ if pendingDependencies == 0 then
104
+ local parentId = getJobIdFromKey(parentKey)
105
+ local parentPrefix = getJobKeyPrefix(parentKey, parentId)
106
+ local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
107
+ if numRemovedElements == 1 then
108
+ if hard then
109
+ if parentPrefix == baseKey then
110
+ removeParentDependencyKey(parentKey, hard, nil, baseKey)
111
+ rcall("DEL", parentKey, parentKey .. ':logs',
112
+ parentKey .. ':dependencies', parentKey .. ':processed')
113
+ else
114
+ moveParentToWait(parentPrefix, parentId)
115
+ end
116
+ else
117
+ moveParentToWait(parentPrefix, parentId, true)
118
+ end
119
+ end
120
+ end
121
+ end
122
+ else
123
+ local missedParentKey = rcall("HGET", jobKey, "parentKey")
124
+ if( (type(missedParentKey) == "string") and missedParentKey ~= "" and (rcall("EXISTS", missedParentKey) == 1)) then
125
+ local parentProcessedKey = missedParentKey .. ":processed"
126
+ rcall("HDEL", parentProcessedKey, jobKey)
127
+ local parentDependenciesKey = missedParentKey .. ":dependencies"
128
+ local result = rcall("SREM", parentDependenciesKey, jobKey)
129
+ if result > 0 then
130
+ local pendingDependencies = rcall("SCARD", parentDependenciesKey)
131
+ if pendingDependencies == 0 then
132
+ local parentId = getJobIdFromKey(missedParentKey)
133
+ local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
134
+ local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
135
+ if numRemovedElements == 1 then
136
+ if hard then
137
+ if parentPrefix == baseKey then
138
+ removeParentDependencyKey(missedParentKey, hard, nil, baseKey)
139
+ rcall("DEL", missedParentKey, missedParentKey .. ':logs',
140
+ missedParentKey .. ':dependencies', missedParentKey .. ':processed')
141
+ else
142
+ moveParentToWait(parentPrefix, parentId)
143
+ end
144
+ else
145
+ moveParentToWait(parentPrefix, parentId, true)
146
+ end
147
+ end
148
+ end
149
+ end
150
+ end
151
+ end
152
+ end
153
+ local function removeJob(jobId, hard, baseKey)
154
+ local jobKey = baseKey .. jobId
155
+ removeParentDependencyKey(jobKey, hard, nil, baseKey)
156
+ rcall("DEL", jobKey, jobKey .. ':logs',
157
+ jobKey .. ':dependencies', jobKey .. ':processed')
158
+ end
159
+ --[[
160
+ Functions to remove jobs by max age.
161
+ ]]
162
+ -- Includes
163
+ local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
164
+ local start = timestamp - maxAge * 1000
165
+ local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
166
+ for i, jobId in ipairs(jobIds) do
167
+ removeJob(jobId, false, prefix)
168
+ end
169
+ rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
170
+ end
171
+ --[[
172
+ Functions to remove jobs by max count.
173
+ ]]
174
+ -- Includes
175
+ local function removeJobsByMaxCount(maxCount, targetSet, prefix)
176
+ local start = maxCount
177
+ local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
178
+ for i, jobId in ipairs(jobIds) do
179
+ removeJob(jobId, false, prefix)
180
+ end
181
+ rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
182
+ end
183
+ --[[
184
+ Function to trim events, default 10000.
185
+ ]]
186
+ local function trimEvents(metaKey, eventStreamKey)
187
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
188
+ if maxEvents ~= false then
189
+ rcall("XTRIM", eventStreamKey, "MAXLEN", "~", maxEvents)
190
+ else
191
+ rcall("XTRIM", eventStreamKey, "MAXLEN", "~", 10000)
192
+ end
193
+ end
194
+ -- Check if we need to check for stalled jobs now.
195
+ local function checkStalledJobs(stalledKey, waitKey, activeKey, failedKey, stalledCheckKey,
196
+ metaKey, pausedKey, eventStreamKey, maxStalledJobCount, queueKeyPrefix, timestamp, maxCheckTime)
197
+ if rcall("EXISTS", stalledCheckKey) == 1 then
198
+ return {{}, {}}
199
+ end
200
+ rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
201
+ -- Trim events before emiting them to avoid trimming events emitted in this script
202
+ trimEvents(metaKey, eventStreamKey)
203
+ -- Move all stalled jobs to wait
204
+ local stalling = rcall('SMEMBERS', stalledKey)
205
+ local stalled = {}
206
+ local failed = {}
207
+ if (#stalling > 0) then
208
+ rcall('DEL', stalledKey)
209
+ local MAX_STALLED_JOB_COUNT = tonumber(maxStalledJobCount)
210
+ -- Remove from active list
211
+ for i, jobId in ipairs(stalling) do
212
+ if jobId == '0' then
213
+ -- If the jobId is a delay marker ID we just remove it.
214
+ local removed = rcall("LREM", activeKey, 1, jobId)
215
+ else
216
+ local jobKey = queueKeyPrefix .. jobId
217
+ -- Check that the lock is also missing, then we can handle this job as really stalled.
218
+ if (rcall("EXISTS", jobKey .. ":lock") == 0) then
219
+ -- Remove from the active queue.
220
+ local removed = rcall("LREM", activeKey, 1, jobId)
221
+ if (removed > 0) then
222
+ -- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
223
+ local stalledCount =
224
+ rcall("HINCRBY", jobKey, "stalledCounter", 1)
225
+ if (stalledCount > MAX_STALLED_JOB_COUNT) then
226
+ local rawOpts = rcall("HGET", jobKey, "opts")
227
+ local opts = cjson.decode(rawOpts)
228
+ local removeOnFailType = type(opts["removeOnFail"])
229
+ rcall("ZADD", failedKey, timestamp, jobId)
230
+ local failedReason =
231
+ "job stalled more than allowable limit"
232
+ rcall("HMSET", jobKey, "failedReason", failedReason,
233
+ "finishedOn", timestamp)
234
+ rcall("XADD", eventStreamKey, "*", "event", "failed", "jobId",
235
+ jobId, 'prev', 'active', 'failedReason',
236
+ failedReason)
237
+ if removeOnFailType == "number" then
238
+ removeJobsByMaxCount(opts["removeOnFail"], failedKey,
239
+ queueKeyPrefix)
240
+ elseif removeOnFailType == "boolean" then
241
+ if opts["removeOnFail"] then
242
+ removeJob(jobId, false, queueKeyPrefix)
243
+ rcall("ZREM", failedKey, jobId)
244
+ end
245
+ elseif removeOnFailType ~= "nil" then
246
+ local maxAge = opts["removeOnFail"]["age"]
247
+ local maxCount = opts["removeOnFail"]["count"]
248
+ if maxAge ~= nil then
249
+ removeJobsByMaxAge(timestamp, maxAge, failedKey,
250
+ queueKeyPrefix)
251
+ end
252
+ if maxCount ~= nil and maxCount > 0 then
253
+ removeJobsByMaxCount(maxCount, failedKey, queueKeyPrefix)
254
+ end
255
+ end
256
+ table.insert(failed, jobId)
257
+ else
258
+ local target = getTargetQueueList(metaKey, waitKey,
259
+ pausedKey)
260
+ -- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
261
+ rcall("RPUSH", target, jobId)
262
+ rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
263
+ jobId, 'prev', 'active')
264
+ -- Emit the stalled event
265
+ rcall("XADD", eventStreamKey, "*", "event", "stalled", "jobId",
266
+ jobId)
267
+ table.insert(stalled, jobId)
268
+ end
269
+ end
270
+ end
271
+ end
272
+ end
273
+ end
274
+ -- Mark potentially stalled jobs
275
+ local active = rcall('LRANGE', activeKey, 0, -1)
276
+ if (#active > 0) then
277
+ for from, to in batches(#active, 7000) do
278
+ rcall('SADD', stalledKey, unpack(active, from, to))
279
+ end
280
+ end
281
+ return {failed, stalled}
282
+ end
283
+ return checkStalledJobs(KEYS[1], KEYS[2], KEYS[3], KEYS[4], KEYS[5], KEYS[6],
284
+ KEYS[7], KEYS[8], ARGV[1], ARGV[2], ARGV[3], ARGV[4])
285
+ `;
286
+ exports.moveStalledJobsToWait = {
287
+ name: 'moveStalledJobsToWait',
288
+ content,
289
+ keys: 8,
290
+ };
291
+ //# sourceMappingURL=moveStalledJobsToWait-8.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"moveStalledJobsToWait-8.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyRf,CAAC;AACW,QAAA,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -0,0 +1,5 @@
1
+ export declare const moveToActive: {
2
+ name: string;
3
+ content: string;
4
+ keys: number;
5
+ };
@@ -0,0 +1,207 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.moveToActive = void 0;
4
+ const content = `--[[
5
+ Move next job to be processed to active, lock it and fetch its data. The job
6
+ may be delayed, in that case we need to move it to the delayed set instead.
7
+ This operation guarantees that the worker owns the job during the lock
8
+ expiration time. The worker is responsible of keeping the lock fresh
9
+ so that no other worker picks this job again.
10
+ Input:
11
+ KEYS[1] wait key
12
+ KEYS[2] active key
13
+ KEYS[3] priority key
14
+ KEYS[4] stream events key
15
+ KEYS[5] stalled key
16
+ -- Rate limiting
17
+ KEYS[6] rate limiter key
18
+ KEYS[7] delayed key
19
+ -- Promote delayed jobs
20
+ KEYS[8] paused key
21
+ KEYS[9] meta key
22
+ -- Arguments
23
+ ARGV[1] key prefix
24
+ ARGV[2] timestamp
25
+ ARGV[3] optional job ID
26
+ ARGV[4] opts
27
+ opts - token - lock token
28
+ opts - lockDuration
29
+ opts - limiter
30
+ ]]
31
+ local jobId
32
+ local rcall = redis.call
33
+ -- Includes
34
+ --[[
35
+ Function to move job from wait state to active.
36
+ Input:
37
+ keys[1] wait key
38
+ keys[2] active key
39
+ keys[3] priority key
40
+ keys[4] stream events key
41
+ keys[5] stalled key
42
+ -- Rate limiting
43
+ keys[6] rate limiter key
44
+ keys[7] delayed key
45
+ opts - token - lock token
46
+ opts - lockDuration
47
+ opts - limiter
48
+ ]]
49
+ local function moveJobFromWaitToActive(keys, keyPrefix, jobId, processedOn, opts)
50
+ -- Check if we need to perform rate limiting.
51
+ local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
52
+ if(maxJobs) then
53
+ local rateLimiterKey = keys[6];
54
+ local groupKey
55
+ local groupKeyOpt = opts['limiter'] and opts['limiter']['groupKey'] or ""
56
+ if groupKeyOpt ~= "" then
57
+ groupKey = string.match(jobId, "[^:]+$")
58
+ if groupKey ~= jobId then
59
+ rateLimiterKey = rateLimiterKey .. ":" .. groupKey
60
+ end
61
+ end
62
+ local jobCounter
63
+ if groupKey ~= nil then
64
+ if rateLimiterKey ~= keys[6] then
65
+ jobCounter = tonumber(rcall("INCR", rateLimiterKey))
66
+ end
67
+ else
68
+ jobCounter = tonumber(rcall("INCR", rateLimiterKey))
69
+ end
70
+ local limiterDuration = opts['limiter'] and opts['limiter']['duration']
71
+ -- check if rate limit hit
72
+ if jobCounter ~= nil and jobCounter > maxJobs then
73
+ local exceedingJobs = jobCounter - maxJobs
74
+ local expireTime = tonumber(rcall("PTTL", rateLimiterKey))
75
+ local delay = expireTime + ((exceedingJobs - 1) * limiterDuration) / maxJobs;
76
+ local timestamp = delay + tonumber(processedOn)
77
+ -- put job into delayed queue
78
+ rcall("ZADD", keys[7], timestamp * 0x1000 + bit.band(jobCounter, 0xfff), jobId);
79
+ rcall("XADD", keys[4], "*", "event", "delayed", "jobId", jobId, "delay", timestamp);
80
+ -- remove from active queue
81
+ rcall("LREM", keys[2], 1, jobId)
82
+ -- Return when we can process more jobs
83
+ return expireTime
84
+ else
85
+ if jobCounter == 1 then
86
+ rcall("PEXPIRE", rateLimiterKey, limiterDuration)
87
+ end
88
+ end
89
+ end
90
+ local jobKey = keyPrefix .. jobId
91
+ local lockKey = jobKey .. ':lock'
92
+ -- get a lock
93
+ if opts['token'] ~= "0" then
94
+ rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
95
+ end
96
+ rcall("ZREM", keys[3], jobId) -- remove from priority
97
+ rcall("XADD", keys[4], "*", "event", "active", "jobId", jobId, "prev", "waiting")
98
+ rcall("HSET", jobKey, "processedOn", processedOn)
99
+ rcall("HINCRBY", jobKey, "attemptsMade", 1)
100
+ return {rcall("HGETALL", jobKey), jobId} -- get job data
101
+ end
102
+ --[[
103
+ Function to return the next delayed job timestamp.
104
+ ]]
105
+ local function getNextDelayedTimestamp(delayedKey)
106
+ local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
107
+ if #result then
108
+ local nextTimestamp = tonumber(result[2])
109
+ if (nextTimestamp ~= nil) then
110
+ nextTimestamp = nextTimestamp / 0x1000
111
+ end
112
+ return nextTimestamp
113
+ end
114
+ end
115
+ --[[
116
+ Updates the delay set, by moving delayed jobs that should
117
+ be processed now to "wait".
118
+ Events:
119
+ 'waiting'
120
+ ]]
121
+ local rcall = redis.call
122
+ -- Includes
123
+ --[[
124
+ Function to add job considering priority.
125
+ ]]
126
+ local function addJobWithPriority(priorityKey, priority, targetKey, jobId)
127
+ rcall("ZADD", priorityKey, priority, jobId)
128
+ local count = rcall("ZCOUNT", priorityKey, 0, priority)
129
+ local len = rcall("LLEN", targetKey)
130
+ local id = rcall("LINDEX", targetKey, len - (count - 1))
131
+ if id then
132
+ rcall("LINSERT", targetKey, "BEFORE", id, jobId)
133
+ else
134
+ rcall("RPUSH", targetKey, jobId)
135
+ end
136
+ end
137
+ --[[
138
+ Function to check for the meta.paused key to decide if we are paused or not
139
+ (since an empty list and !EXISTS are not really the same).
140
+ ]]
141
+ local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
142
+ if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
143
+ return waitKey
144
+ else
145
+ return pausedKey
146
+ end
147
+ end
148
+ -- Try to get as much as 1000 jobs at once, and returns the nextTimestamp if
149
+ -- there are more delayed jobs to process.
150
+ local function promoteDelayedJobs(delayedKey, waitKey, priorityKey, pausedKey,
151
+ metaKey, eventStreamKey, prefix, timestamp)
152
+ local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000, "LIMIT", 0, 1000)
153
+ if (#jobs > 0) then
154
+ rcall("ZREM", delayedKey, unpack(jobs))
155
+ -- check if we need to use push in paused instead of waiting
156
+ local target = getTargetQueueList(metaKey, waitKey, pausedKey)
157
+ for _, jobId in ipairs(jobs) do
158
+ local priority =
159
+ tonumber(rcall("HGET", prefix .. jobId, "priority")) or 0
160
+ if priority == 0 then
161
+ -- LIFO or FIFO
162
+ rcall("LPUSH", target, jobId)
163
+ else
164
+ addJobWithPriority(priorityKey, priority, target, jobId)
165
+ end
166
+ -- Emit waiting event
167
+ rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
168
+ jobId, "prev", "delayed")
169
+ rcall("HSET", prefix .. jobId, "delay", 0)
170
+ end
171
+ end
172
+ local nextTimestamp = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")[2]
173
+ if (nextTimestamp ~= nil) then
174
+ nextTimestamp = nextTimestamp / 0x1000
175
+ end
176
+ return nextTimestamp
177
+ end
178
+ -- Check if there are delayed jobs that we can move to wait.
179
+ promoteDelayedJobs(KEYS[7], KEYS[1], KEYS[3], KEYS[8], KEYS[9], KEYS[4], ARGV[1], ARGV[2])
180
+ if (ARGV[3] ~= "") then
181
+ jobId = ARGV[3]
182
+ -- clean stalled key
183
+ rcall("SREM", KEYS[5], jobId)
184
+ else
185
+ -- no job ID, try non-blocking move from wait to active
186
+ jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
187
+ end
188
+ -- If jobId is special ID 0, then there is no job to process
189
+ if jobId == "0" then
190
+ rcall("LREM", KEYS[2], 1, 0)
191
+ elseif jobId then
192
+ local opts = cmsgpack.unpack(ARGV[4])
193
+ -- this script is not really moving, it is preparing the job for processing
194
+ return moveJobFromWaitToActive(KEYS, ARGV[1], jobId, ARGV[2], opts)
195
+ end
196
+ -- Return the timestamp for the next delayed job if any.
197
+ local nextTimestamp = getNextDelayedTimestamp(KEYS[7])
198
+ if (nextTimestamp ~= nil) then
199
+ return nextTimestamp - tonumber(ARGV[2])
200
+ end
201
+ `;
202
+ exports.moveToActive = {
203
+ name: 'moveToActive',
204
+ content,
205
+ keys: 9,
206
+ };
207
+ //# sourceMappingURL=moveToActive-9.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"moveToActive-9.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqMf,CAAC;AACW,QAAA,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -0,0 +1,5 @@
1
+ export declare const moveToDelayed: {
2
+ name: string;
3
+ content: string;
4
+ keys: number;
5
+ };
@@ -1,29 +1,26 @@
1
- --[[
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.moveToDelayed = void 0;
4
+ const content = `--[[
2
5
  Moves job from active to delayed set.
3
-
4
6
  Input:
5
7
  KEYS[1] active key
6
8
  KEYS[2] delayed key
7
9
  KEYS[3] job key
8
10
  KEYS[4] events stream
9
11
  KEYS[5] delayed stream
10
-
11
12
  ARGV[1] delayedTimestamp
12
13
  ARGV[2] the id of the job
13
14
  ARGV[3] queue token
14
-
15
15
  Output:
16
16
  0 - OK
17
17
  -1 - Missing job.
18
18
  -3 - Job not in active set.
19
-
20
19
  Events:
21
20
  - delayed key.
22
21
  ]]
23
22
  local rcall = redis.call
24
-
25
23
  if rcall("EXISTS", KEYS[3]) == 1 then
26
-
27
24
  if ARGV[3] ~= "0" then
28
25
  local lockKey = KEYS[3] .. ':lock'
29
26
  if rcall("GET", lockKey) == ARGV[3] then
@@ -32,23 +29,24 @@ if rcall("EXISTS", KEYS[3]) == 1 then
32
29
  return -2
33
30
  end
34
31
  end
35
-
36
32
  local jobId = ARGV[2]
37
33
  local score = tonumber(ARGV[1])
38
34
  local delayedTimestamp = (score / 0x1000)
39
-
40
35
  local numRemovedElements = rcall("LREM", KEYS[1], -1, jobId)
41
-
42
36
  if(numRemovedElements < 1) then
43
37
  return -3
44
38
  end
45
-
46
39
  rcall("ZADD", KEYS[2], score, jobId)
47
-
48
40
  rcall("XADD", KEYS[4], "*", "event", "delayed", "jobId", jobId, "delay", delayedTimestamp);
49
41
  rcall("XADD", KEYS[5], "*", "nextTimestamp", delayedTimestamp);
50
-
51
42
  return 0
52
43
  else
53
44
  return -1
54
45
  end
46
+ `;
47
+ exports.moveToDelayed = {
48
+ name: 'moveToDelayed',
49
+ content,
50
+ keys: 5,
51
+ };
52
+ //# sourceMappingURL=moveToDelayed-5.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"moveToDelayed-5.js","sourceRoot":"","sources":["../../../src/scripts/moveToDelayed-5.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0Cf,CAAC;AACW,QAAA,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -0,0 +1,5 @@
1
+ export declare const moveToFinished: {
2
+ name: string;
3
+ content: string;
4
+ keys: number;
5
+ };