bullmq 2.1.3 → 2.2.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (244) hide show
  1. package/dist/cjs/classes/flow-producer.js +10 -0
  2. package/dist/cjs/classes/flow-producer.js.map +1 -1
  3. package/dist/cjs/classes/job.d.ts +9 -1
  4. package/dist/cjs/classes/job.js +10 -0
  5. package/dist/cjs/classes/job.js.map +1 -1
  6. package/dist/cjs/classes/redis-connection.d.ts +6 -2
  7. package/dist/cjs/classes/redis-connection.js +12 -6
  8. package/dist/cjs/classes/redis-connection.js.map +1 -1
  9. package/dist/cjs/interfaces/flow-job.d.ts +6 -3
  10. package/dist/cjs/scripts/addJob-8.d.ts +5 -0
  11. package/dist/{esm/commands/addJob-8.lua → cjs/scripts/addJob-8.js} +85 -32
  12. package/dist/cjs/scripts/addJob-8.js.map +1 -0
  13. package/dist/cjs/scripts/changeDelay-4.d.ts +5 -0
  14. package/dist/cjs/{commands/changeDelay-4.lua → scripts/changeDelay-4.js} +11 -11
  15. package/dist/cjs/scripts/changeDelay-4.js.map +1 -0
  16. package/dist/cjs/scripts/cleanJobsInSet-2.d.ts +5 -0
  17. package/dist/cjs/scripts/cleanJobsInSet-2.js +264 -0
  18. package/dist/cjs/scripts/cleanJobsInSet-2.js.map +1 -0
  19. package/dist/cjs/scripts/drain-4.d.ts +5 -0
  20. package/dist/cjs/{commands/includes/removeParentDependencyKey.lua → scripts/drain-4.js} +110 -10
  21. package/dist/cjs/scripts/drain-4.js.map +1 -0
  22. package/dist/cjs/scripts/extendLock-2.d.ts +5 -0
  23. package/dist/{esm/commands/extendLock-2.lua → cjs/scripts/extendLock-2.js} +11 -4
  24. package/dist/cjs/scripts/extendLock-2.js.map +1 -0
  25. package/dist/cjs/scripts/getState-7.d.ts +5 -0
  26. package/dist/{esm/commands/getState-7.lua → cjs/scripts/getState-7.js} +22 -12
  27. package/dist/cjs/scripts/getState-7.js.map +1 -0
  28. package/dist/cjs/scripts/getStateV2-7.d.ts +5 -0
  29. package/dist/cjs/{commands/getStateV2-7.lua → scripts/getStateV2-7.js} +11 -10
  30. package/dist/cjs/scripts/getStateV2-7.js.map +1 -0
  31. package/dist/cjs/scripts/index.d.ts +26 -0
  32. package/dist/cjs/scripts/index.js +30 -0
  33. package/dist/cjs/scripts/index.js.map +1 -0
  34. package/dist/cjs/scripts/isFinished-3.d.ts +5 -0
  35. package/dist/cjs/{commands/isFinished-3.lua → scripts/isFinished-3.js} +11 -10
  36. package/dist/cjs/scripts/isFinished-3.js.map +1 -0
  37. package/dist/cjs/scripts/isJobInList-1.d.ts +5 -0
  38. package/dist/cjs/scripts/isJobInList-1.js +32 -0
  39. package/dist/cjs/scripts/isJobInList-1.js.map +1 -0
  40. package/dist/cjs/scripts/moveStalledJobsToWait-8.d.ts +5 -0
  41. package/dist/cjs/scripts/moveStalledJobsToWait-8.js +291 -0
  42. package/dist/cjs/scripts/moveStalledJobsToWait-8.js.map +1 -0
  43. package/dist/cjs/scripts/moveToActive-9.d.ts +5 -0
  44. package/dist/cjs/scripts/moveToActive-9.js +207 -0
  45. package/dist/cjs/scripts/moveToActive-9.js.map +1 -0
  46. package/dist/cjs/scripts/moveToDelayed-5.d.ts +5 -0
  47. package/dist/cjs/{commands/moveToDelayed-5.lua → scripts/moveToDelayed-5.js} +11 -13
  48. package/dist/cjs/scripts/moveToDelayed-5.js.map +1 -0
  49. package/dist/cjs/scripts/moveToFinished-12.d.ts +5 -0
  50. package/dist/cjs/scripts/moveToFinished-12.js +538 -0
  51. package/dist/cjs/scripts/moveToFinished-12.js.map +1 -0
  52. package/dist/cjs/scripts/moveToWaitingChildren-4.d.ts +5 -0
  53. package/dist/cjs/{commands/moveToWaitingChildren-4.lua → scripts/moveToWaitingChildren-4.js} +11 -14
  54. package/dist/cjs/scripts/moveToWaitingChildren-4.js.map +1 -0
  55. package/dist/cjs/scripts/obliterate-2.d.ts +5 -0
  56. package/dist/cjs/scripts/obliterate-2.js +241 -0
  57. package/dist/cjs/scripts/obliterate-2.js.map +1 -0
  58. package/dist/cjs/scripts/pause-4.d.ts +5 -0
  59. package/dist/{esm/commands/pause-4.lua → cjs/scripts/pause-4.js} +11 -7
  60. package/dist/cjs/scripts/pause-4.js.map +1 -0
  61. package/dist/cjs/scripts/promote-6.d.ts +5 -0
  62. package/dist/cjs/{commands/promote-6.lua → scripts/promote-6.js} +22 -14
  63. package/dist/cjs/scripts/promote-6.js.map +1 -0
  64. package/dist/cjs/scripts/releaseLock-1.d.ts +5 -0
  65. package/dist/{esm/commands/releaseLock-1.lua → cjs/scripts/releaseLock-1.js} +11 -5
  66. package/dist/cjs/scripts/releaseLock-1.js.map +1 -0
  67. package/dist/cjs/scripts/removeJob-1.d.ts +5 -0
  68. package/dist/cjs/scripts/removeJob-1.js +206 -0
  69. package/dist/cjs/scripts/removeJob-1.js.map +1 -0
  70. package/dist/cjs/scripts/removeRepeatable-2.d.ts +5 -0
  71. package/dist/cjs/{commands/removeRepeatable-2.lua → scripts/removeRepeatable-2.js} +11 -7
  72. package/dist/cjs/scripts/removeRepeatable-2.js.map +1 -0
  73. package/dist/cjs/scripts/reprocessJob-4.d.ts +5 -0
  74. package/dist/{esm/commands/reprocessJob-4.lua → cjs/scripts/reprocessJob-4.js} +11 -5
  75. package/dist/cjs/scripts/reprocessJob-4.js.map +1 -0
  76. package/dist/cjs/scripts/retryJob-6.d.ts +5 -0
  77. package/dist/{esm/commands/retryJob-6.lua → cjs/scripts/retryJob-6.js} +22 -13
  78. package/dist/cjs/scripts/retryJob-6.js.map +1 -0
  79. package/dist/cjs/scripts/retryJobs-6.d.ts +5 -0
  80. package/dist/cjs/{commands/retryJobs-6.lua → scripts/retryJobs-6.js} +38 -15
  81. package/dist/cjs/scripts/retryJobs-6.js.map +1 -0
  82. package/dist/cjs/scripts/takeLock-1.d.ts +5 -0
  83. package/dist/cjs/{commands/takeLock-1.lua → scripts/takeLock-1.js} +11 -4
  84. package/dist/cjs/scripts/takeLock-1.js.map +1 -0
  85. package/dist/cjs/scripts/updateData-1.d.ts +5 -0
  86. package/dist/cjs/scripts/updateData-1.js +23 -0
  87. package/dist/cjs/scripts/updateData-1.js.map +1 -0
  88. package/dist/cjs/scripts/updateProgress-2.d.ts +5 -0
  89. package/dist/{esm/commands/updateProgress-2.lua → cjs/scripts/updateProgress-2.js} +11 -5
  90. package/dist/cjs/scripts/updateProgress-2.js.map +1 -0
  91. package/dist/esm/classes/flow-producer.js +10 -0
  92. package/dist/esm/classes/flow-producer.js.map +1 -1
  93. package/dist/esm/classes/job.d.ts +9 -1
  94. package/dist/esm/classes/job.js +10 -0
  95. package/dist/esm/classes/job.js.map +1 -1
  96. package/dist/esm/classes/redis-connection.d.ts +6 -2
  97. package/dist/esm/classes/redis-connection.js +12 -6
  98. package/dist/esm/classes/redis-connection.js.map +1 -1
  99. package/dist/esm/interfaces/flow-job.d.ts +6 -3
  100. package/dist/esm/scripts/addJob-8.d.ts +5 -0
  101. package/dist/{cjs/commands/addJob-8.lua → esm/scripts/addJob-8.js} +82 -32
  102. package/dist/esm/scripts/addJob-8.js.map +1 -0
  103. package/dist/esm/scripts/changeDelay-4.d.ts +5 -0
  104. package/dist/esm/{commands/changeDelay-4.lua → scripts/changeDelay-4.js} +8 -11
  105. package/dist/esm/scripts/changeDelay-4.js.map +1 -0
  106. package/dist/esm/scripts/cleanJobsInSet-2.d.ts +5 -0
  107. package/dist/esm/scripts/cleanJobsInSet-2.js +261 -0
  108. package/dist/esm/scripts/cleanJobsInSet-2.js.map +1 -0
  109. package/dist/esm/scripts/drain-4.d.ts +5 -0
  110. package/dist/esm/{commands/includes/removeParentDependencyKey.lua → scripts/drain-4.js} +107 -10
  111. package/dist/esm/scripts/drain-4.js.map +1 -0
  112. package/dist/esm/scripts/extendLock-2.d.ts +5 -0
  113. package/dist/{cjs/commands/extendLock-2.lua → esm/scripts/extendLock-2.js} +8 -4
  114. package/dist/esm/scripts/extendLock-2.js.map +1 -0
  115. package/dist/esm/scripts/getState-7.d.ts +5 -0
  116. package/dist/{cjs/commands/getState-7.lua → esm/scripts/getState-7.js} +19 -12
  117. package/dist/esm/scripts/getState-7.js.map +1 -0
  118. package/dist/esm/scripts/getStateV2-7.d.ts +5 -0
  119. package/dist/esm/{commands/getStateV2-7.lua → scripts/getStateV2-7.js} +8 -10
  120. package/dist/esm/scripts/getStateV2-7.js.map +1 -0
  121. package/dist/esm/scripts/index.d.ts +26 -0
  122. package/dist/esm/scripts/index.js +27 -0
  123. package/dist/esm/scripts/index.js.map +1 -0
  124. package/dist/esm/scripts/isFinished-3.d.ts +5 -0
  125. package/dist/esm/{commands/isFinished-3.lua → scripts/isFinished-3.js} +8 -10
  126. package/dist/esm/scripts/isFinished-3.js.map +1 -0
  127. package/dist/esm/scripts/isJobInList-1.d.ts +5 -0
  128. package/dist/esm/scripts/isJobInList-1.js +29 -0
  129. package/dist/esm/scripts/isJobInList-1.js.map +1 -0
  130. package/dist/esm/scripts/moveStalledJobsToWait-8.d.ts +5 -0
  131. package/dist/esm/scripts/moveStalledJobsToWait-8.js +288 -0
  132. package/dist/esm/scripts/moveStalledJobsToWait-8.js.map +1 -0
  133. package/dist/esm/scripts/moveToActive-9.d.ts +5 -0
  134. package/dist/esm/scripts/moveToActive-9.js +204 -0
  135. package/dist/esm/scripts/moveToActive-9.js.map +1 -0
  136. package/dist/esm/scripts/moveToDelayed-5.d.ts +5 -0
  137. package/dist/esm/{commands/moveToDelayed-5.lua → scripts/moveToDelayed-5.js} +8 -13
  138. package/dist/esm/scripts/moveToDelayed-5.js.map +1 -0
  139. package/dist/esm/scripts/moveToFinished-12.d.ts +5 -0
  140. package/dist/esm/scripts/moveToFinished-12.js +535 -0
  141. package/dist/esm/scripts/moveToFinished-12.js.map +1 -0
  142. package/dist/esm/scripts/moveToWaitingChildren-4.d.ts +5 -0
  143. package/dist/esm/{commands/moveToWaitingChildren-4.lua → scripts/moveToWaitingChildren-4.js} +8 -14
  144. package/dist/esm/scripts/moveToWaitingChildren-4.js.map +1 -0
  145. package/dist/esm/scripts/obliterate-2.d.ts +5 -0
  146. package/dist/esm/scripts/obliterate-2.js +238 -0
  147. package/dist/esm/scripts/obliterate-2.js.map +1 -0
  148. package/dist/esm/scripts/pause-4.d.ts +5 -0
  149. package/dist/{cjs/commands/pause-4.lua → esm/scripts/pause-4.js} +8 -7
  150. package/dist/esm/scripts/pause-4.js.map +1 -0
  151. package/dist/esm/scripts/promote-6.d.ts +5 -0
  152. package/dist/esm/{commands/promote-6.lua → scripts/promote-6.js} +19 -14
  153. package/dist/esm/scripts/promote-6.js.map +1 -0
  154. package/dist/esm/scripts/releaseLock-1.d.ts +5 -0
  155. package/dist/{cjs/commands/releaseLock-1.lua → esm/scripts/releaseLock-1.js} +8 -5
  156. package/dist/esm/scripts/releaseLock-1.js.map +1 -0
  157. package/dist/esm/scripts/removeJob-1.d.ts +5 -0
  158. package/dist/esm/scripts/removeJob-1.js +203 -0
  159. package/dist/esm/scripts/removeJob-1.js.map +1 -0
  160. package/dist/esm/scripts/removeRepeatable-2.d.ts +5 -0
  161. package/dist/esm/{commands/removeRepeatable-2.lua → scripts/removeRepeatable-2.js} +8 -7
  162. package/dist/esm/scripts/removeRepeatable-2.js.map +1 -0
  163. package/dist/esm/scripts/reprocessJob-4.d.ts +5 -0
  164. package/dist/{cjs/commands/reprocessJob-4.lua → esm/scripts/reprocessJob-4.js} +8 -5
  165. package/dist/esm/scripts/reprocessJob-4.js.map +1 -0
  166. package/dist/esm/scripts/retryJob-6.d.ts +5 -0
  167. package/dist/{cjs/commands/retryJob-6.lua → esm/scripts/retryJob-6.js} +19 -13
  168. package/dist/esm/scripts/retryJob-6.js.map +1 -0
  169. package/dist/esm/scripts/retryJobs-6.d.ts +5 -0
  170. package/dist/esm/{commands/retryJobs-6.lua → scripts/retryJobs-6.js} +35 -15
  171. package/dist/esm/scripts/retryJobs-6.js.map +1 -0
  172. package/dist/esm/scripts/takeLock-1.d.ts +5 -0
  173. package/dist/esm/{commands/takeLock-1.lua → scripts/takeLock-1.js} +8 -4
  174. package/dist/esm/scripts/takeLock-1.js.map +1 -0
  175. package/dist/esm/scripts/updateData-1.d.ts +5 -0
  176. package/dist/esm/{commands/updateData-1.lua → scripts/updateData-1.js} +8 -4
  177. package/dist/esm/scripts/updateData-1.js.map +1 -0
  178. package/dist/esm/scripts/updateProgress-2.d.ts +5 -0
  179. package/dist/{cjs/commands/updateProgress-2.lua → esm/scripts/updateProgress-2.js} +8 -5
  180. package/dist/esm/scripts/updateProgress-2.js.map +1 -0
  181. package/package.json +4 -2
  182. package/dist/cjs/commands/cleanJobsInSet-2.lua +0 -46
  183. package/dist/cjs/commands/drain-4.lua +0 -25
  184. package/dist/cjs/commands/includes/addJobWithPriority.lua +0 -16
  185. package/dist/cjs/commands/includes/batches.lua +0 -18
  186. package/dist/cjs/commands/includes/checkItemInList.lua +0 -12
  187. package/dist/cjs/commands/includes/checkStalledJobs.lua +0 -136
  188. package/dist/cjs/commands/includes/cleanList.lua +0 -48
  189. package/dist/cjs/commands/includes/cleanSet.lua +0 -50
  190. package/dist/cjs/commands/includes/collectMetrics.lua +0 -46
  191. package/dist/cjs/commands/includes/destructureJobKey.lua +0 -12
  192. package/dist/cjs/commands/includes/getNextDelayedTimestamp.lua +0 -13
  193. package/dist/cjs/commands/includes/getTargetQueueList.lua +0 -12
  194. package/dist/cjs/commands/includes/getTimestamp.lua +0 -19
  195. package/dist/cjs/commands/includes/getZSetItems.lua +0 -7
  196. package/dist/cjs/commands/includes/isLocked.lua +0 -31
  197. package/dist/cjs/commands/includes/moveJobFromWaitToActive.lua +0 -84
  198. package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +0 -25
  199. package/dist/cjs/commands/includes/promoteDelayedJobs.lua +0 -49
  200. package/dist/cjs/commands/includes/removeJob.lua +0 -13
  201. package/dist/cjs/commands/includes/removeJobFromAnyState.lua +0 -26
  202. package/dist/cjs/commands/includes/removeJobs.lua +0 -38
  203. package/dist/cjs/commands/includes/removeJobsByMaxAge.lua +0 -15
  204. package/dist/cjs/commands/includes/removeJobsByMaxCount.lua +0 -15
  205. package/dist/cjs/commands/includes/trimEvents.lua +0 -12
  206. package/dist/cjs/commands/includes/updateParentDepsIfNeeded.lua +0 -27
  207. package/dist/cjs/commands/isJobInList-1.lua +0 -16
  208. package/dist/cjs/commands/moveStalledJobsToWait-8.lua +0 -24
  209. package/dist/cjs/commands/moveToActive-9.lua +0 -67
  210. package/dist/cjs/commands/moveToFinished-12.lua +0 -200
  211. package/dist/cjs/commands/obliterate-2.lua +0 -94
  212. package/dist/cjs/commands/removeJob-1.lua +0 -72
  213. package/dist/cjs/commands/updateData-1.lua +0 -16
  214. package/dist/esm/commands/cleanJobsInSet-2.lua +0 -46
  215. package/dist/esm/commands/drain-4.lua +0 -25
  216. package/dist/esm/commands/includes/addJobWithPriority.lua +0 -16
  217. package/dist/esm/commands/includes/batches.lua +0 -18
  218. package/dist/esm/commands/includes/checkItemInList.lua +0 -12
  219. package/dist/esm/commands/includes/checkStalledJobs.lua +0 -136
  220. package/dist/esm/commands/includes/cleanList.lua +0 -48
  221. package/dist/esm/commands/includes/cleanSet.lua +0 -50
  222. package/dist/esm/commands/includes/collectMetrics.lua +0 -46
  223. package/dist/esm/commands/includes/destructureJobKey.lua +0 -12
  224. package/dist/esm/commands/includes/getNextDelayedTimestamp.lua +0 -13
  225. package/dist/esm/commands/includes/getTargetQueueList.lua +0 -12
  226. package/dist/esm/commands/includes/getTimestamp.lua +0 -19
  227. package/dist/esm/commands/includes/getZSetItems.lua +0 -7
  228. package/dist/esm/commands/includes/isLocked.lua +0 -31
  229. package/dist/esm/commands/includes/moveJobFromWaitToActive.lua +0 -84
  230. package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +0 -25
  231. package/dist/esm/commands/includes/promoteDelayedJobs.lua +0 -49
  232. package/dist/esm/commands/includes/removeJob.lua +0 -13
  233. package/dist/esm/commands/includes/removeJobFromAnyState.lua +0 -26
  234. package/dist/esm/commands/includes/removeJobs.lua +0 -38
  235. package/dist/esm/commands/includes/removeJobsByMaxAge.lua +0 -15
  236. package/dist/esm/commands/includes/removeJobsByMaxCount.lua +0 -15
  237. package/dist/esm/commands/includes/trimEvents.lua +0 -12
  238. package/dist/esm/commands/includes/updateParentDepsIfNeeded.lua +0 -27
  239. package/dist/esm/commands/isJobInList-1.lua +0 -16
  240. package/dist/esm/commands/moveStalledJobsToWait-8.lua +0 -24
  241. package/dist/esm/commands/moveToActive-9.lua +0 -67
  242. package/dist/esm/commands/moveToFinished-12.lua +0 -200
  243. package/dist/esm/commands/obliterate-2.lua +0 -94
  244. package/dist/esm/commands/removeJob-1.lua +0 -72
@@ -1,200 +0,0 @@
1
- --[[
2
- Move job from active to a finished status (completed o failed)
3
- A job can only be moved to completed if it was active.
4
- The job must be locked before it can be moved to a finished status,
5
- and the lock must be released in this script.
6
-
7
- Input:
8
- KEYS[1] wait key
9
- KEYS[2] active key
10
- KEYS[3] priority key
11
- KEYS[4] event stream key
12
- KEYS[5] stalled key
13
-
14
- -- Rate limiting
15
- KEYS[6] rate limiter key
16
- KEYS[7] delayed key
17
-
18
- KEYS[8] paused key
19
-
20
- KEYS[9] completed/failed key
21
- KEYS[10] jobId key
22
- KEYS[11] meta key
23
- KEYS[12] metrics key
24
-
25
- ARGV[1] jobId
26
- ARGV[2] timestamp
27
- ARGV[3] msg property returnvalue / failedReason
28
- ARGV[4] return value / failed reason
29
- ARGV[5] target (completed/failed)
30
- ARGV[6] event data (? maybe just send jobid).
31
- ARGV[7] fetch next?
32
- ARGV[8] keys prefix
33
- ARGV[9] opts
34
-
35
- opts - token - lock token
36
- opts - keepJobs
37
- opts - lockDuration - lock duration in milliseconds
38
- opts - parent - parent data
39
- opts - parentKey
40
- opts - attempts max attempts
41
- opts - attemptsMade
42
- opts - maxMetricsSize
43
- opts - fpof - fail parent on fail
44
-
45
- Output:
46
- 0 OK
47
- -1 Missing key.
48
- -2 Missing lock.
49
- -3 Job not in active set
50
- -4 Job has pending dependencies
51
-
52
- Events:
53
- 'completed/failed'
54
- ]]
55
- local rcall = redis.call
56
-
57
- --- Includes
58
- --- @include "includes/collectMetrics"
59
- --- @include "includes/getNextDelayedTimestamp"
60
- --- @include "includes/moveJobFromWaitToActive"
61
- --- @include "includes/moveParentFromWaitingChildrenToFailed"
62
- --- @include "includes/promoteDelayedJobs"
63
- --- @include "includes/removeJobsByMaxAge"
64
- --- @include "includes/removeJobsByMaxCount"
65
- --- @include "includes/trimEvents"
66
- --- @include "includes/updateParentDepsIfNeeded"
67
-
68
- local jobIdKey = KEYS[10]
69
- if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
70
- local opts = cmsgpack.unpack(ARGV[9])
71
-
72
- local token = opts['token']
73
- local parentId = opts['parent'] and opts['parent']['id'] or ""
74
- local parentQueueKey = opts['parent'] and opts['parent']['queue'] or ""
75
- local parentKey = opts['parentKey'] or ""
76
- local attempts = opts['attempts']
77
- local attemptsMade = opts['attemptsMade']
78
- local maxMetricsSize = opts['maxMetricsSize']
79
- local maxCount = opts['keepJobs']['count']
80
- local maxAge = opts['keepJobs']['age']
81
-
82
- if token ~= "0" then
83
- local lockKey = jobIdKey .. ':lock'
84
- if rcall("GET", lockKey) == token then
85
- rcall("DEL", lockKey)
86
- rcall("SREM", KEYS[5], ARGV[1])
87
- else
88
- return -2
89
- end
90
- end
91
-
92
- if rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then -- // Make sure it does not have pending dependencies
93
- return -4
94
- end
95
-
96
- local jobId = ARGV[1]
97
- local timestamp = ARGV[2]
98
-
99
- -- Remove from active list (if not active we shall return error)
100
- local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
101
-
102
- if (numRemovedElements < 1) then return -3 end
103
-
104
- -- Trim events before emiting them to avoid trimming events emitted in this script
105
- trimEvents(KEYS[11], KEYS[4])
106
-
107
- -- If job has a parent we need to
108
- -- 1) remove this job id from parents dependencies
109
- -- 2) move the job Id to parent "processed" set
110
- -- 3) push the results into parent "results" list
111
- -- 4) if parent's dependencies is empty, then move parent to "wait/paused". Note it may be a different queue!.
112
- if parentId == "" and parentKey ~= "" then
113
- parentId = getJobIdFromKey(parentKey)
114
- parentQueueKey = getJobKeyPrefix(parentKey, ":" .. parentId)
115
- end
116
- if parentId ~= "" then
117
- if ARGV[5] == "completed" then
118
- local dependenciesSet = parentKey .. ":dependencies"
119
- if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
120
- updateParentDepsIfNeeded(parentKey, parentQueueKey, dependenciesSet,
121
- parentId, jobIdKey, ARGV[4])
122
- end
123
- elseif opts['fpof'] then
124
- moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
125
- end
126
- end
127
-
128
- -- Remove job?
129
- if maxCount ~= 0 then
130
- local targetSet = KEYS[9]
131
- -- Add to complete/failed set
132
- rcall("ZADD", targetSet, timestamp, jobId)
133
- rcall("HMSET", jobIdKey, ARGV[3], ARGV[4], "finishedOn", timestamp) -- "returnvalue" / "failedReason" and "finishedOn"
134
-
135
- -- Remove old jobs?
136
- local prefix = ARGV[8]
137
-
138
- if maxAge ~= nil then
139
- removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
140
- end
141
-
142
- if maxCount ~= nil and maxCount > 0 then
143
- removeJobsByMaxCount(maxCount, targetSet, prefix)
144
- end
145
- else
146
- rcall("DEL", jobIdKey, jobIdKey .. ':logs', jobIdKey .. ':processed')
147
- end
148
-
149
- rcall("XADD", KEYS[4], "*", "event", ARGV[5], "jobId", jobId, ARGV[3],
150
- ARGV[4])
151
-
152
- if ARGV[5] == "failed" then
153
- if tonumber(attemptsMade) >= tonumber(attempts) then
154
- rcall("XADD", KEYS[4], "*", "event", "retries-exhausted", "jobId",
155
- jobId, "attemptsMade", attemptsMade)
156
- end
157
- end
158
-
159
- -- Collect metrics
160
- if maxMetricsSize ~= "" then
161
- collectMetrics(KEYS[12], KEYS[12]..':data', maxMetricsSize, timestamp)
162
- end
163
-
164
- -- Try to get next job to avoid an extra roundtrip if the queue is not closing,
165
- -- and not rate limited.
166
- if (ARGV[7] == "1") then
167
-
168
- -- Check if there are delayed jobs that can be promoted
169
- promoteDelayedJobs(KEYS[7], KEYS[1], KEYS[3], KEYS[8], KEYS[11], KEYS[4], ARGV[8], timestamp)
170
-
171
- jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
172
-
173
- if jobId == "0" then
174
- rcall("LREM", KEYS[2], 1, 0)
175
- elseif jobId then
176
- return moveJobFromWaitToActive(KEYS, ARGV[8], jobId, timestamp, opts)
177
- end
178
-
179
- -- Return the timestamp for the next delayed job if any.
180
- local nextTimestamp = getNextDelayedTimestamp(KEYS[7])
181
- if (nextTimestamp ~= nil) then
182
- -- The result is guaranteed to be positive, since the
183
- -- ZRANGEBYSCORE command would have return a job otherwise.
184
- return nextTimestamp - timestamp
185
- end
186
- end
187
-
188
- local waitLen = rcall("LLEN", KEYS[1])
189
- if waitLen == 0 then
190
- local activeLen = rcall("LLEN", KEYS[2])
191
-
192
- if activeLen == 0 then
193
- rcall("XADD", KEYS[4], "*", "event", "drained")
194
- end
195
- end
196
-
197
- return 0
198
- else
199
- return -1
200
- end
@@ -1,94 +0,0 @@
1
- --[[
2
- Completely obliterates a queue and all of its contents
3
-
4
- Input:
5
- KEYS[1] meta
6
- KEYS[2] base
7
-
8
- ARGV[1] count
9
- ARGV[2] force
10
- ]]
11
-
12
- -- This command completely destroys a queue including all of its jobs, current or past
13
- -- leaving no trace of its existence. Since this script needs to iterate to find all the job
14
- -- keys, consider that this call may be slow for very large queues.
15
-
16
- -- The queue needs to be "paused" or it will return an error
17
- -- If the queue has currently active jobs then the script by default will return error,
18
- -- however this behaviour can be overrided using the 'force' option.
19
- local maxCount = tonumber(ARGV[1])
20
- local baseKey = KEYS[2]
21
-
22
- local rcall = redis.call
23
-
24
- --- @include "includes/removeJobs"
25
-
26
- local function removeLockKeys(keys)
27
- for i, key in ipairs(keys) do
28
- rcall("DEL", baseKey .. key .. ':lock')
29
- end
30
- end
31
-
32
- -- 1) Check if paused, if not return with error.
33
- if rcall("HEXISTS", KEYS[1], "paused") ~= 1 then
34
- return -1 -- Error, NotPaused
35
- end
36
-
37
- -- 2) Check if there are active jobs, if there are and not "force" return error.
38
- local activeKey = baseKey .. 'active'
39
- local activeJobs = getListItems(activeKey, maxCount)
40
- if (#activeJobs > 0) then
41
- if(ARGV[2] == "") then
42
- return -2 -- Error, ExistActiveJobs
43
- end
44
- end
45
-
46
- removeLockKeys(activeJobs)
47
- maxCount = removeJobs(activeJobs, true, baseKey, maxCount)
48
- rcall("LTRIM", activeKey, #activeJobs, -1)
49
- if(maxCount <= 0) then
50
- return 1
51
- end
52
-
53
- local delayedKey = baseKey .. 'delayed'
54
- maxCount = removeZSetJobs(delayedKey, true, baseKey, maxCount)
55
- if(maxCount <= 0) then
56
- return 1
57
- end
58
-
59
- local completedKey = baseKey .. 'completed'
60
- maxCount = removeZSetJobs(completedKey, true, baseKey, maxCount)
61
- if(maxCount <= 0) then
62
- return 1
63
- end
64
-
65
- local waitKey = baseKey .. 'paused'
66
- maxCount = removeListJobs(waitKey, true, baseKey, maxCount)
67
- if(maxCount <= 0) then
68
- return 1
69
- end
70
-
71
- local failedKey = baseKey .. 'failed'
72
- maxCount = removeZSetJobs(failedKey, true, baseKey, maxCount)
73
- if(maxCount <= 0) then
74
- return 1
75
- end
76
-
77
- if(maxCount > 0) then
78
- rcall("DEL",
79
- baseKey .. 'priority',
80
- baseKey .. 'events',
81
- baseKey .. 'delay',
82
- baseKey .. 'stalled-check',
83
- baseKey .. 'stalled',
84
- baseKey .. 'id',
85
- baseKey .. 'meta',
86
- baseKey .. 'repeat',
87
- baseKey .. 'metrics:completed',
88
- baseKey .. 'metrics:completed:data',
89
- baseKey .. 'metrics:failed',
90
- baseKey .. 'metrics:failed:data')
91
- return 0
92
- else
93
- return 1
94
- end
@@ -1,72 +0,0 @@
1
- --[[
2
- Remove a job from all the queues it may be in as well as all its data.
3
- In order to be able to remove a job, it cannot be active.
4
-
5
- Input:
6
- KEYS[1] queue prefix
7
- ARGV[1] jobId
8
-
9
- Events:
10
- 'removed'
11
- ]]
12
-
13
- local rcall = redis.call
14
-
15
- -- Includes
16
- --- @include "includes/destructureJobKey"
17
- --- @include "includes/isLocked"
18
- --- @include "includes/removeJobFromAnyState"
19
- --- @include "includes/removeParentDependencyKey"
20
-
21
- local function removeJob( prefix, jobId, parentKey)
22
- local jobKey = prefix .. jobId;
23
-
24
- removeParentDependencyKey(jobKey, false, parentKey)
25
-
26
- -- Check if this job has children
27
- -- If so, we are going to try to remove the children recursively in deep first way because
28
- -- if some job is locked we must exit with and error.
29
- --local countProcessed = rcall("HLEN", jobKey .. ":processed")
30
- local processed = rcall("HGETALL", jobKey .. ":processed")
31
-
32
- if (#processed > 0) then
33
- for i = 1, #processed, 2 do
34
- local childJobId = getJobIdFromKey(processed[i])
35
- local childJobPrefix = getJobKeyPrefix(processed[i], childJobId)
36
- removeJob( childJobPrefix, childJobId, jobKey )
37
- end
38
- end
39
-
40
- local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
41
- if (#dependencies > 0) then
42
- for i, childJobKey in ipairs(dependencies) do
43
- -- We need to get the jobId for this job.
44
- local childJobId = getJobIdFromKey(childJobKey)
45
- local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
46
- removeJob( childJobPrefix, childJobId, jobKey )
47
- end
48
- end
49
-
50
- local prev = removeJobFromAnyState(prefix, jobId)
51
-
52
- rcall("ZREM", prefix .. "priority", jobId)
53
- rcall("DEL", jobKey, jobKey .. ":logs", jobKey .. ":dependencies", jobKey .. ":processed")
54
-
55
- -- -- delete keys related to rate limiter
56
- -- local limiterIndexTable = KEYS[10] .. ":index"
57
- -- local limitedSetKey = rcall("HGET", limiterIndexTable, jobId)
58
- -- if limitedSetKey then
59
- -- rcall("SREM", limitedSetKey, jobId)
60
- -- rcall("HDEL", limiterIndexTable, jobId)
61
- -- end
62
-
63
- rcall("XADD", prefix .. "events", "*", "event", "removed", "jobId", jobId, "prev", prev);
64
- end
65
-
66
- local prefix = KEYS[1]
67
-
68
- if not isLocked(prefix, ARGV[1]) then
69
- removeJob(prefix, ARGV[1])
70
- return 1
71
- end
72
- return 0
@@ -1,16 +0,0 @@
1
- --[[
2
- Update job data
3
-
4
- Input:
5
- KEYS[1] Job id key
6
-
7
- ARGV[1] data
8
- ]]
9
- local rcall = redis.call
10
-
11
- if rcall("EXISTS",KEYS[1]) == 1 then -- // Make sure job exists
12
- rcall("HSET", KEYS[1], "data", ARGV[1])
13
- return 0
14
- else
15
- return -1
16
- end
@@ -1,46 +0,0 @@
1
- --[[
2
- Remove jobs from the specific set.
3
-
4
- Input:
5
- KEYS[1] set key,
6
- KEYS[2] events stream key
7
-
8
- ARGV[1] jobKey prefix
9
- ARGV[2] timestamp
10
- ARGV[3] limit the number of jobs to be removed. 0 is unlimited
11
- ARGV[4] set name, can be any of 'wait', 'active', 'paused', 'delayed', 'completed', or 'failed'
12
- ]]
13
- local rcall = redis.call
14
- local rangeStart = 0
15
- local rangeEnd = -1
16
-
17
- local limit = tonumber(ARGV[3])
18
-
19
- -- If we're only deleting _n_ items, avoid retrieving all items
20
- -- for faster performance
21
- --
22
- -- Start from the tail of the list, since that's where oldest elements
23
- -- are generally added for FIFO lists
24
- if limit > 0 then
25
- rangeStart = -1 - limit + 1
26
- rangeEnd = -1
27
- end
28
-
29
- -- Includes
30
- --- @include "includes/cleanList"
31
- --- @include "includes/cleanSet"
32
-
33
- local result
34
- if ARGV[4] == "active" then
35
- result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], false)
36
- elseif ARGV[4] == "delayed" then
37
- result = cleanSet(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], limit, {"processedOn", "timestamp"})
38
- elseif ARGV[4] == "wait" or ARGV[4] == "paused" then
39
- result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], true)
40
- else
41
- result = cleanSet(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], limit, {"finishedOn"} )
42
- end
43
-
44
- rcall("XADD", KEYS[2], "*", "event", "cleaned", "count", result[2])
45
-
46
- return result[1]
@@ -1,25 +0,0 @@
1
- --[[
2
- Drains the queue, removes all jobs that are waiting
3
- or delayed, but not active, completed or failed
4
-
5
- Input:
6
- KEYS[1] 'wait',
7
- KEYS[2] 'paused'
8
- KEYS[3] 'delayed'
9
- KEYS[4] 'priority'
10
-
11
- ARGV[1] queue key prefix
12
- ]]
13
- local rcall = redis.call
14
- local queueBaseKey = ARGV[1]
15
-
16
- --- @include "includes/removeJobs"
17
-
18
- removeListJobs(KEYS[1], true, queueBaseKey, 0) --wait
19
- removeListJobs(KEYS[2], true, queueBaseKey, 0) --paused
20
-
21
- if KEYS[3] ~= "" then
22
- removeZSetJobs(KEYS[3], true, queueBaseKey, 0) --delayed
23
- end
24
-
25
- rcall("DEL", KEYS[4])
@@ -1,16 +0,0 @@
1
- --[[
2
- Function to add job considering priority.
3
- ]]
4
-
5
- local function addJobWithPriority(priorityKey, priority, targetKey, jobId)
6
- rcall("ZADD", priorityKey, priority, jobId)
7
- local count = rcall("ZCOUNT", priorityKey, 0, priority)
8
-
9
- local len = rcall("LLEN", targetKey)
10
- local id = rcall("LINDEX", targetKey, len - (count - 1))
11
- if id then
12
- rcall("LINSERT", targetKey, "BEFORE", id, jobId)
13
- else
14
- rcall("RPUSH", targetKey, jobId)
15
- end
16
- end
@@ -1,18 +0,0 @@
1
- --[[
2
- Function to loop in batches.
3
- Just a bit of warning, some commands as ZREM
4
- could receive a maximum of 7000 parameters per call.
5
- ]]
6
-
7
- local function batches(n, batchSize)
8
- local i = 0
9
-
10
- return function()
11
- local from = i * batchSize + 1
12
- i = i + 1
13
- if (from <= n) then
14
- local to = math.min(from + batchSize - 1, n)
15
- return from, to
16
- end
17
- end
18
- end
@@ -1,12 +0,0 @@
1
- --[[
2
- Functions to check if a item belongs to a list.
3
- ]]
4
-
5
- local function checkItemInList(list, item)
6
- for _, v in pairs(list) do
7
- if v == item then
8
- return 1
9
- end
10
- end
11
- return nil
12
- end
@@ -1,136 +0,0 @@
1
- --[[
2
- Move stalled jobs to wait.
3
-
4
- Input:
5
- stalledKey 'stalled' (SET)
6
- waitKey 'wait', (LIST)
7
- activeKey 'active', (LIST)
8
- failedKey 'failed', (ZSET)
9
- stalledCheckKey 'stalled-check', (KEY)
10
- metaKey 'meta', (KEY)
11
- pausedKey 'paused', (LIST)
12
- eventStreamKey 'event stream' (STREAM)
13
-
14
- maxStalledJobCount Max stalled job count
15
- queueKeyPrefix queue.toKey('')
16
- timestamp timestamp
17
- maxCheckTime max check time
18
-
19
- Events:
20
- 'stalled' with stalled job id.
21
- ]] local rcall = redis.call
22
-
23
- -- Includes
24
- --- @include "batches"
25
- --- @include "getTargetQueueList"
26
- --- @include "removeJob"
27
- --- @include "removeJobsByMaxAge"
28
- --- @include "removeJobsByMaxCount"
29
- --- @include "trimEvents"
30
-
31
- -- Check if we need to check for stalled jobs now.
32
-
33
- local function checkStalledJobs(stalledKey, waitKey, activeKey, failedKey, stalledCheckKey, metaKey, pausedKey, eventStreamKey, maxStalledJobCount, queueKeyPrefix, timestamp, maxCheckTime)
34
- if rcall("EXISTS", stalledCheckKey) == 1 then
35
- return {{}, {}}
36
- end
37
-
38
- rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
39
-
40
- -- Trim events before emiting them to avoid trimming events emitted in this script
41
- trimEvents(metaKey, eventStreamKey)
42
-
43
- -- Move all stalled jobs to wait
44
- local stalling = rcall('SMEMBERS', stalledKey)
45
- local stalled = {}
46
- local failed = {}
47
- if (#stalling > 0) then
48
- rcall('DEL', stalledKey)
49
-
50
- local MAX_STALLED_JOB_COUNT = tonumber(maxStalledJobCount)
51
-
52
- -- Remove from active list
53
- for i, jobId in ipairs(stalling) do
54
-
55
- if jobId == '0' then
56
- -- If the jobId is a delay marker ID we just remove it.
57
- local removed = rcall("LREM", activeKey, 1, jobId)
58
- else
59
- local jobKey = queueKeyPrefix .. jobId
60
-
61
- -- Check that the lock is also missing, then we can handle this job as really stalled.
62
- if (rcall("EXISTS", jobKey .. ":lock") == 0) then
63
- -- Remove from the active queue.
64
- local removed = rcall("LREM", activeKey, 1, jobId)
65
-
66
- if (removed > 0) then
67
- -- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
68
- local stalledCount =
69
- rcall("HINCRBY", jobKey, "stalledCounter", 1)
70
- if (stalledCount > MAX_STALLED_JOB_COUNT) then
71
- local rawOpts = rcall("HGET", jobKey, "opts")
72
- local opts = cjson.decode(rawOpts)
73
- local removeOnFailType = type(opts["removeOnFail"])
74
- rcall("ZADD", failedKey, timestamp, jobId)
75
- local failedReason =
76
- "job stalled more than allowable limit"
77
- rcall("HMSET", jobKey, "failedReason", failedReason,
78
- "finishedOn", timestamp)
79
- rcall("XADD", eventStreamKey, "*", "event", "failed", "jobId",
80
- jobId, 'prev', 'active', 'failedReason',
81
- failedReason)
82
-
83
- if removeOnFailType == "number" then
84
- removeJobsByMaxCount(opts["removeOnFail"], failedKey,
85
- queueKeyPrefix)
86
- elseif removeOnFailType == "boolean" then
87
- if opts["removeOnFail"] then
88
- removeJob(jobId, false, queueKeyPrefix)
89
- rcall("ZREM", failedKey, jobId)
90
- end
91
- elseif removeOnFailType ~= "nil" then
92
- local maxAge = opts["removeOnFail"]["age"]
93
- local maxCount = opts["removeOnFail"]["count"]
94
-
95
- if maxAge ~= nil then
96
- removeJobsByMaxAge(timestamp, maxAge, failedKey,
97
- queueKeyPrefix)
98
- end
99
-
100
- if maxCount ~= nil and maxCount > 0 then
101
- removeJobsByMaxCount(maxCount, failedKey, queueKeyPrefix)
102
- end
103
- end
104
-
105
- table.insert(failed, jobId)
106
- else
107
- local target = getTargetQueueList(metaKey, waitKey,
108
- pausedKey)
109
-
110
- -- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
111
- rcall("RPUSH", target, jobId)
112
- rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
113
- jobId, 'prev', 'active')
114
-
115
- -- Emit the stalled event
116
- rcall("XADD", eventStreamKey, "*", "event", "stalled", "jobId",
117
- jobId)
118
- table.insert(stalled, jobId)
119
- end
120
- end
121
- end
122
- end
123
- end
124
- end
125
-
126
- -- Mark potentially stalled jobs
127
- local active = rcall('LRANGE', activeKey, 0, -1)
128
-
129
- if (#active > 0) then
130
- for from, to in batches(#active, 7000) do
131
- rcall('SADD', stalledKey, unpack(active, from, to))
132
- end
133
- end
134
-
135
- return {failed, stalled}
136
- end
@@ -1,48 +0,0 @@
1
- --[[
2
- Function to clean job list.
3
- Returns jobIds and deleted count number.
4
- ]]
5
-
6
- -- Includes
7
- --- @include "getTimestamp"
8
- --- @include "removeJob"
9
-
10
- local function cleanList(listKey, jobKeyPrefix, rangeStart, rangeEnd,
11
- timestamp, isWaiting)
12
- local jobs = rcall("LRANGE", listKey, rangeStart, rangeEnd)
13
- local deleted = {}
14
- local deletedCount = 0
15
- local jobTS
16
- local deletionMarker = ''
17
- local jobIdsLen = #jobs
18
- for i, job in ipairs(jobs) do
19
- if limit > 0 and deletedCount >= limit then
20
- break
21
- end
22
-
23
- local jobKey = jobKeyPrefix .. job
24
- if (isWaiting or rcall("EXISTS", jobKey .. ":lock") == 0) then
25
- -- Find the right timestamp of the job to compare to maxTimestamp:
26
- -- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
27
- -- * processedOn represents when the job was last attempted, but it doesn't get populated until the job is first tried
28
- -- * timestamp is the original job submission time
29
- -- Fetch all three of these (in that order) and use the first one that is set so that we'll leave jobs that have been active within the grace period:
30
- jobTS = getTimestamp(jobKey, {"finishedOn", "processedOn", "timestamp"})
31
- if (not jobTS or jobTS < timestamp) then
32
- -- replace the entry with a deletion marker; the actual deletion will
33
- -- occur at the end of the script
34
- rcall("LSET", listKey, rangeEnd - jobIdsLen + i, deletionMarker)
35
- removeJob(job, true, jobKeyPrefix)
36
- if isWaiting then
37
- rcall("ZREM", jobKeyPrefix .. "priority", job)
38
- end
39
- deletedCount = deletedCount + 1
40
- table.insert(deleted, job)
41
- end
42
- end
43
- end
44
-
45
- rcall("LREM", listKey, 0, deletionMarker)
46
-
47
- return {deleted, deletedCount}
48
- end