bullmq 2.3.0 → 2.3.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (109) hide show
  1. package/dist/cjs/classes/redis-connection.js +2 -2
  2. package/dist/cjs/classes/redis-connection.js.map +1 -1
  3. package/dist/cjs/commands/addJob-8.lua +174 -0
  4. package/dist/cjs/commands/changeDelay-4.lua +43 -0
  5. package/dist/cjs/commands/cleanJobsInSet-2.lua +46 -0
  6. package/dist/cjs/commands/drain-4.lua +25 -0
  7. package/dist/cjs/commands/extendLock-2.lua +23 -0
  8. package/dist/cjs/commands/getState-7.lua +57 -0
  9. package/dist/cjs/commands/getStateV2-7.lua +51 -0
  10. package/dist/cjs/commands/includes/addJobWithPriority.lua +16 -0
  11. package/dist/cjs/commands/includes/batches.lua +18 -0
  12. package/dist/cjs/commands/includes/checkItemInList.lua +12 -0
  13. package/dist/cjs/commands/includes/checkStalledJobs.lua +137 -0
  14. package/dist/cjs/commands/includes/cleanList.lua +50 -0
  15. package/dist/cjs/commands/includes/cleanSet.lua +50 -0
  16. package/dist/cjs/commands/includes/collectMetrics.lua +46 -0
  17. package/dist/cjs/commands/includes/destructureJobKey.lua +12 -0
  18. package/dist/cjs/commands/includes/getNextDelayedTimestamp.lua +13 -0
  19. package/dist/cjs/commands/includes/getTargetQueueList.lua +12 -0
  20. package/dist/cjs/commands/includes/getTimestamp.lua +19 -0
  21. package/dist/cjs/commands/includes/getZSetItems.lua +7 -0
  22. package/dist/cjs/commands/includes/isLocked.lua +31 -0
  23. package/dist/cjs/commands/includes/moveJobFromWaitToActive.lua +84 -0
  24. package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +28 -0
  25. package/dist/cjs/commands/includes/promoteDelayedJobs.lua +49 -0
  26. package/dist/cjs/commands/includes/removeJob.lua +13 -0
  27. package/dist/cjs/commands/includes/removeJobFromAnyState.lua +26 -0
  28. package/dist/cjs/commands/includes/removeJobs.lua +38 -0
  29. package/dist/cjs/commands/includes/removeJobsByMaxAge.lua +15 -0
  30. package/dist/cjs/commands/includes/removeJobsByMaxCount.lua +15 -0
  31. package/dist/cjs/commands/includes/removeParentDependencyKey.lua +81 -0
  32. package/dist/cjs/commands/includes/trimEvents.lua +12 -0
  33. package/dist/cjs/commands/includes/updateParentDepsIfNeeded.lua +28 -0
  34. package/dist/cjs/commands/isFinished-3.lua +48 -0
  35. package/dist/cjs/commands/isJobInList-1.lua +16 -0
  36. package/dist/cjs/commands/moveStalledJobsToWait-8.lua +24 -0
  37. package/dist/cjs/commands/moveToActive-9.lua +67 -0
  38. package/dist/cjs/commands/moveToDelayed-5.lua +54 -0
  39. package/dist/cjs/commands/moveToFinished-12.lua +201 -0
  40. package/dist/cjs/commands/moveToWaitingChildren-4.lua +62 -0
  41. package/dist/cjs/commands/obliterate-2.lua +94 -0
  42. package/dist/cjs/commands/pause-4.lua +27 -0
  43. package/dist/cjs/commands/promote-6.lua +59 -0
  44. package/dist/cjs/commands/releaseLock-1.lua +19 -0
  45. package/dist/cjs/commands/removeJob-1.lua +72 -0
  46. package/dist/cjs/commands/removeRepeatable-2.lua +32 -0
  47. package/dist/cjs/commands/reprocessJob-4.lua +35 -0
  48. package/dist/cjs/commands/retryJob-6.lua +51 -0
  49. package/dist/cjs/commands/retryJobs-6.lua +53 -0
  50. package/dist/cjs/commands/takeLock-1.lua +17 -0
  51. package/dist/cjs/commands/updateData-1.lua +16 -0
  52. package/dist/cjs/commands/updateProgress-2.lua +22 -0
  53. package/dist/cjs/scripts/moveToFinished-12.js +5 -2
  54. package/dist/cjs/scripts/moveToFinished-12.js.map +1 -1
  55. package/dist/esm/classes/redis-connection.js +2 -2
  56. package/dist/esm/classes/redis-connection.js.map +1 -1
  57. package/dist/esm/commands/addJob-8.lua +174 -0
  58. package/dist/esm/commands/changeDelay-4.lua +43 -0
  59. package/dist/esm/commands/cleanJobsInSet-2.lua +46 -0
  60. package/dist/esm/commands/drain-4.lua +25 -0
  61. package/dist/esm/commands/extendLock-2.lua +23 -0
  62. package/dist/esm/commands/getState-7.lua +57 -0
  63. package/dist/esm/commands/getStateV2-7.lua +51 -0
  64. package/dist/esm/commands/includes/addJobWithPriority.lua +16 -0
  65. package/dist/esm/commands/includes/batches.lua +18 -0
  66. package/dist/esm/commands/includes/checkItemInList.lua +12 -0
  67. package/dist/esm/commands/includes/checkStalledJobs.lua +137 -0
  68. package/dist/esm/commands/includes/cleanList.lua +50 -0
  69. package/dist/esm/commands/includes/cleanSet.lua +50 -0
  70. package/dist/esm/commands/includes/collectMetrics.lua +46 -0
  71. package/dist/esm/commands/includes/destructureJobKey.lua +12 -0
  72. package/dist/esm/commands/includes/getNextDelayedTimestamp.lua +13 -0
  73. package/dist/esm/commands/includes/getTargetQueueList.lua +12 -0
  74. package/dist/esm/commands/includes/getTimestamp.lua +19 -0
  75. package/dist/esm/commands/includes/getZSetItems.lua +7 -0
  76. package/dist/esm/commands/includes/isLocked.lua +31 -0
  77. package/dist/esm/commands/includes/moveJobFromWaitToActive.lua +84 -0
  78. package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +28 -0
  79. package/dist/esm/commands/includes/promoteDelayedJobs.lua +49 -0
  80. package/dist/esm/commands/includes/removeJob.lua +13 -0
  81. package/dist/esm/commands/includes/removeJobFromAnyState.lua +26 -0
  82. package/dist/esm/commands/includes/removeJobs.lua +38 -0
  83. package/dist/esm/commands/includes/removeJobsByMaxAge.lua +15 -0
  84. package/dist/esm/commands/includes/removeJobsByMaxCount.lua +15 -0
  85. package/dist/esm/commands/includes/removeParentDependencyKey.lua +81 -0
  86. package/dist/esm/commands/includes/trimEvents.lua +12 -0
  87. package/dist/esm/commands/includes/updateParentDepsIfNeeded.lua +28 -0
  88. package/dist/esm/commands/isFinished-3.lua +48 -0
  89. package/dist/esm/commands/isJobInList-1.lua +16 -0
  90. package/dist/esm/commands/moveStalledJobsToWait-8.lua +24 -0
  91. package/dist/esm/commands/moveToActive-9.lua +67 -0
  92. package/dist/esm/commands/moveToDelayed-5.lua +54 -0
  93. package/dist/esm/commands/moveToFinished-12.lua +201 -0
  94. package/dist/esm/commands/moveToWaitingChildren-4.lua +62 -0
  95. package/dist/esm/commands/obliterate-2.lua +94 -0
  96. package/dist/esm/commands/pause-4.lua +27 -0
  97. package/dist/esm/commands/promote-6.lua +59 -0
  98. package/dist/esm/commands/releaseLock-1.lua +19 -0
  99. package/dist/esm/commands/removeJob-1.lua +72 -0
  100. package/dist/esm/commands/removeRepeatable-2.lua +32 -0
  101. package/dist/esm/commands/reprocessJob-4.lua +35 -0
  102. package/dist/esm/commands/retryJob-6.lua +51 -0
  103. package/dist/esm/commands/retryJobs-6.lua +53 -0
  104. package/dist/esm/commands/takeLock-1.lua +17 -0
  105. package/dist/esm/commands/updateData-1.lua +16 -0
  106. package/dist/esm/commands/updateProgress-2.lua +22 -0
  107. package/dist/esm/scripts/moveToFinished-12.js +5 -2
  108. package/dist/esm/scripts/moveToFinished-12.js.map +1 -1
  109. package/package.json +2 -2
@@ -0,0 +1,201 @@
1
+ --[[
2
+ Move job from active to a finished status (completed o failed)
3
+ A job can only be moved to completed if it was active.
4
+ The job must be locked before it can be moved to a finished status,
5
+ and the lock must be released in this script.
6
+
7
+ Input:
8
+ KEYS[1] wait key
9
+ KEYS[2] active key
10
+ KEYS[3] priority key
11
+ KEYS[4] event stream key
12
+ KEYS[5] stalled key
13
+
14
+ -- Rate limiting
15
+ KEYS[6] rate limiter key
16
+ KEYS[7] delayed key
17
+
18
+ KEYS[8] paused key
19
+
20
+ KEYS[9] completed/failed key
21
+ KEYS[10] jobId key
22
+ KEYS[11] meta key
23
+ KEYS[12] metrics key
24
+
25
+ ARGV[1] jobId
26
+ ARGV[2] timestamp
27
+ ARGV[3] msg property returnvalue / failedReason
28
+ ARGV[4] return value / failed reason
29
+ ARGV[5] target (completed/failed)
30
+ ARGV[6] event data (? maybe just send jobid).
31
+ ARGV[7] fetch next?
32
+ ARGV[8] keys prefix
33
+ ARGV[9] opts
34
+
35
+ opts - token - lock token
36
+ opts - keepJobs
37
+ opts - lockDuration - lock duration in milliseconds
38
+ opts - parent - parent data
39
+ opts - parentKey
40
+ opts - attempts max attempts
41
+ opts - attemptsMade
42
+ opts - maxMetricsSize
43
+ opts - fpof - fail parent on fail
44
+
45
+ Output:
46
+ 0 OK
47
+ -1 Missing key.
48
+ -2 Missing lock.
49
+ -3 Job not in active set
50
+ -4 Job has pending dependencies
51
+
52
+ Events:
53
+ 'completed/failed'
54
+ ]]
55
+ local rcall = redis.call
56
+
57
+ --- Includes
58
+ --- @include "includes/collectMetrics"
59
+ --- @include "includes/getNextDelayedTimestamp"
60
+ --- @include "includes/moveJobFromWaitToActive"
61
+ --- @include "includes/moveParentFromWaitingChildrenToFailed"
62
+ --- @include "includes/promoteDelayedJobs"
63
+ --- @include "includes/removeJobsByMaxAge"
64
+ --- @include "includes/removeJobsByMaxCount"
65
+ --- @include "includes/trimEvents"
66
+ --- @include "includes/updateParentDepsIfNeeded"
67
+
68
+ local jobIdKey = KEYS[10]
69
+ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
70
+ local opts = cmsgpack.unpack(ARGV[9])
71
+
72
+ local token = opts['token']
73
+ local parentId = opts['parent'] and opts['parent']['id'] or ""
74
+ local parentQueueKey = opts['parent'] and opts['parent']['queue'] or ""
75
+ local parentKey = opts['parentKey'] or ""
76
+ local attempts = opts['attempts']
77
+ local attemptsMade = opts['attemptsMade']
78
+ local maxMetricsSize = opts['maxMetricsSize']
79
+ local maxCount = opts['keepJobs']['count']
80
+ local maxAge = opts['keepJobs']['age']
81
+
82
+ if token ~= "0" then
83
+ local lockKey = jobIdKey .. ':lock'
84
+ if rcall("GET", lockKey) == token then
85
+ rcall("DEL", lockKey)
86
+ rcall("SREM", KEYS[5], ARGV[1])
87
+ else
88
+ return -2
89
+ end
90
+ end
91
+
92
+ if rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then -- // Make sure it does not have pending dependencies
93
+ return -4
94
+ end
95
+
96
+ local jobId = ARGV[1]
97
+ local timestamp = ARGV[2]
98
+
99
+ -- Remove from active list (if not active we shall return error)
100
+ local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
101
+
102
+ if (numRemovedElements < 1) then return -3 end
103
+
104
+ -- Trim events before emiting them to avoid trimming events emitted in this script
105
+ trimEvents(KEYS[11], KEYS[4])
106
+
107
+ -- If job has a parent we need to
108
+ -- 1) remove this job id from parents dependencies
109
+ -- 2) move the job Id to parent "processed" set
110
+ -- 3) push the results into parent "results" list
111
+ -- 4) if parent's dependencies is empty, then move parent to "wait/paused". Note it may be a different queue!.
112
+ if parentId == "" and parentKey ~= "" then
113
+ parentId = getJobIdFromKey(parentKey)
114
+ parentQueueKey = getJobKeyPrefix(parentKey, ":" .. parentId)
115
+ end
116
+ if parentId ~= "" then
117
+ if ARGV[5] == "completed" then
118
+ local dependenciesSet = parentKey .. ":dependencies"
119
+ if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
120
+ updateParentDepsIfNeeded(parentKey, parentQueueKey, dependenciesSet,
121
+ parentId, jobIdKey, ARGV[4])
122
+ end
123
+ elseif opts['fpof'] then
124
+ moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
125
+ end
126
+ end
127
+
128
+ -- Remove job?
129
+ if maxCount ~= 0 then
130
+ local targetSet = KEYS[9]
131
+ -- Add to complete/failed set
132
+ rcall("ZADD", targetSet, timestamp, jobId)
133
+ rcall("HMSET", jobIdKey, ARGV[3], ARGV[4], "finishedOn", timestamp)
134
+ -- "returnvalue" / "failedReason" and "finishedOn"
135
+
136
+ -- Remove old jobs?
137
+ local prefix = ARGV[8]
138
+
139
+ if maxAge ~= nil then
140
+ removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
141
+ end
142
+
143
+ if maxCount ~= nil and maxCount > 0 then
144
+ removeJobsByMaxCount(maxCount, targetSet, prefix)
145
+ end
146
+ else
147
+ rcall("DEL", jobIdKey, jobIdKey .. ':logs', jobIdKey .. ':processed')
148
+ end
149
+
150
+ rcall("XADD", KEYS[4], "*", "event", ARGV[5], "jobId", jobId, ARGV[3],
151
+ ARGV[4])
152
+
153
+ if ARGV[5] == "failed" then
154
+ if tonumber(attemptsMade) >= tonumber(attempts) then
155
+ rcall("XADD", KEYS[4], "*", "event", "retries-exhausted", "jobId",
156
+ jobId, "attemptsMade", attemptsMade)
157
+ end
158
+ end
159
+
160
+ -- Collect metrics
161
+ if maxMetricsSize ~= "" then
162
+ collectMetrics(KEYS[12], KEYS[12]..':data', maxMetricsSize, timestamp)
163
+ end
164
+
165
+ -- Try to get next job to avoid an extra roundtrip if the queue is not closing,
166
+ -- and not rate limited.
167
+ if (ARGV[7] == "1") then
168
+
169
+ -- Check if there are delayed jobs that can be promoted
170
+ promoteDelayedJobs(KEYS[7], KEYS[1], KEYS[3], KEYS[8], KEYS[11], KEYS[4], ARGV[8], timestamp)
171
+
172
+ jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
173
+
174
+ if jobId == "0" then
175
+ rcall("LREM", KEYS[2], 1, 0)
176
+ elseif jobId then
177
+ return moveJobFromWaitToActive(KEYS, ARGV[8], jobId, timestamp, opts)
178
+ end
179
+
180
+ -- Return the timestamp for the next delayed job if any.
181
+ local nextTimestamp = getNextDelayedTimestamp(KEYS[7])
182
+ if (nextTimestamp ~= nil) then
183
+ -- The result is guaranteed to be positive, since the
184
+ -- ZRANGEBYSCORE command would have return a job otherwise.
185
+ return nextTimestamp - timestamp
186
+ end
187
+ end
188
+
189
+ local waitLen = rcall("LLEN", KEYS[1])
190
+ if waitLen == 0 then
191
+ local activeLen = rcall("LLEN", KEYS[2])
192
+
193
+ if activeLen == 0 then
194
+ rcall("XADD", KEYS[4], "*", "event", "drained")
195
+ end
196
+ end
197
+
198
+ return 0
199
+ else
200
+ return -1
201
+ end
@@ -0,0 +1,62 @@
1
+ --[[
2
+ Moves job from active to waiting children set.
3
+
4
+ Input:
5
+ KEYS[1] lock key
6
+ KEYS[2] active key
7
+ KEYS[3] waitChildrenKey key
8
+ KEYS[4] job key
9
+
10
+ ARGV[1] token
11
+ ARGV[2] child key
12
+ ARGV[3] timestamp
13
+ ARGV[4] the id of the job
14
+
15
+ Output:
16
+ 0 - OK
17
+ 1 - There are not pending dependencies.
18
+ -1 - Missing job.
19
+ -2 - Missing lock
20
+ -3 - Job not in active set
21
+ ]]
22
+ local rcall = redis.call
23
+
24
+ local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId, timestamp, lockKey, token)
25
+ if token ~= "0" then
26
+ if rcall("GET", lockKey) == token then
27
+ rcall("DEL", lockKey)
28
+ else
29
+ return -2
30
+ end
31
+ end
32
+
33
+ local score = tonumber(timestamp)
34
+
35
+ local numRemovedElements = rcall("LREM", activeKey, -1, jobId)
36
+
37
+ if(numRemovedElements < 1) then
38
+ return -3
39
+ end
40
+
41
+ rcall("ZADD", waitingChildrenKey, score, jobId)
42
+
43
+ return 0
44
+ end
45
+
46
+ if rcall("EXISTS", KEYS[4]) == 1 then
47
+ if ARGV[2] ~= "" then
48
+ if rcall("SISMEMBER", KEYS[4] .. ":dependencies", ARGV[2]) ~= 0 then
49
+ return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], ARGV[1])
50
+ end
51
+
52
+ return 1
53
+ else
54
+ if rcall("SCARD", KEYS[4] .. ":dependencies") ~= 0 then
55
+ return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], ARGV[1])
56
+ end
57
+
58
+ return 1
59
+ end
60
+ end
61
+
62
+ return -1
@@ -0,0 +1,94 @@
1
+ --[[
2
+ Completely obliterates a queue and all of its contents
3
+
4
+ Input:
5
+ KEYS[1] meta
6
+ KEYS[2] base
7
+
8
+ ARGV[1] count
9
+ ARGV[2] force
10
+ ]]
11
+
12
+ -- This command completely destroys a queue including all of its jobs, current or past
13
+ -- leaving no trace of its existence. Since this script needs to iterate to find all the job
14
+ -- keys, consider that this call may be slow for very large queues.
15
+
16
+ -- The queue needs to be "paused" or it will return an error
17
+ -- If the queue has currently active jobs then the script by default will return error,
18
+ -- however this behaviour can be overrided using the 'force' option.
19
+ local maxCount = tonumber(ARGV[1])
20
+ local baseKey = KEYS[2]
21
+
22
+ local rcall = redis.call
23
+
24
+ --- @include "includes/removeJobs"
25
+
26
+ local function removeLockKeys(keys)
27
+ for i, key in ipairs(keys) do
28
+ rcall("DEL", baseKey .. key .. ':lock')
29
+ end
30
+ end
31
+
32
+ -- 1) Check if paused, if not return with error.
33
+ if rcall("HEXISTS", KEYS[1], "paused") ~= 1 then
34
+ return -1 -- Error, NotPaused
35
+ end
36
+
37
+ -- 2) Check if there are active jobs, if there are and not "force" return error.
38
+ local activeKey = baseKey .. 'active'
39
+ local activeJobs = getListItems(activeKey, maxCount)
40
+ if (#activeJobs > 0) then
41
+ if(ARGV[2] == "") then
42
+ return -2 -- Error, ExistActiveJobs
43
+ end
44
+ end
45
+
46
+ removeLockKeys(activeJobs)
47
+ maxCount = removeJobs(activeJobs, true, baseKey, maxCount)
48
+ rcall("LTRIM", activeKey, #activeJobs, -1)
49
+ if(maxCount <= 0) then
50
+ return 1
51
+ end
52
+
53
+ local delayedKey = baseKey .. 'delayed'
54
+ maxCount = removeZSetJobs(delayedKey, true, baseKey, maxCount)
55
+ if(maxCount <= 0) then
56
+ return 1
57
+ end
58
+
59
+ local completedKey = baseKey .. 'completed'
60
+ maxCount = removeZSetJobs(completedKey, true, baseKey, maxCount)
61
+ if(maxCount <= 0) then
62
+ return 1
63
+ end
64
+
65
+ local waitKey = baseKey .. 'paused'
66
+ maxCount = removeListJobs(waitKey, true, baseKey, maxCount)
67
+ if(maxCount <= 0) then
68
+ return 1
69
+ end
70
+
71
+ local failedKey = baseKey .. 'failed'
72
+ maxCount = removeZSetJobs(failedKey, true, baseKey, maxCount)
73
+ if(maxCount <= 0) then
74
+ return 1
75
+ end
76
+
77
+ if(maxCount > 0) then
78
+ rcall("DEL",
79
+ baseKey .. 'priority',
80
+ baseKey .. 'events',
81
+ baseKey .. 'delay',
82
+ baseKey .. 'stalled-check',
83
+ baseKey .. 'stalled',
84
+ baseKey .. 'id',
85
+ baseKey .. 'meta',
86
+ baseKey .. 'repeat',
87
+ baseKey .. 'metrics:completed',
88
+ baseKey .. 'metrics:completed:data',
89
+ baseKey .. 'metrics:failed',
90
+ baseKey .. 'metrics:failed:data')
91
+ return 0
92
+ else
93
+ return 1
94
+ end
@@ -0,0 +1,27 @@
1
+ --[[
2
+ Pauses or resumes a queue globably.
3
+
4
+ Input:
5
+ KEYS[1] 'wait' or 'paused''
6
+ KEYS[2] 'paused' or 'wait'
7
+ KEYS[3] 'meta'
8
+ KEYS[4] events stream key
9
+
10
+ ARGV[1] 'paused' or 'resumed'
11
+
12
+ Event:
13
+ publish paused or resumed event.
14
+ ]]
15
+ local rcall = redis.call
16
+
17
+ if rcall("EXISTS", KEYS[1]) == 1 then
18
+ rcall("RENAME", KEYS[1], KEYS[2])
19
+ end
20
+
21
+ if ARGV[1] == "paused" then
22
+ rcall("HSET", KEYS[3], "paused", 1)
23
+ else
24
+ rcall("HDEL", KEYS[3], "paused")
25
+ end
26
+
27
+ rcall("XADD", KEYS[4], "*", "event", ARGV[1]);
@@ -0,0 +1,59 @@
1
+ --[[
2
+ Promotes a job that is currently "delayed" to the "waiting" state
3
+
4
+ Input:
5
+ KEYS[1] 'delayed'
6
+ KEYS[2] 'wait'
7
+ KEYS[3] 'paused'
8
+ KEYS[4] 'meta'
9
+ KEYS[5] 'priority'
10
+ KEYS[6] 'event stream'
11
+
12
+ ARGV[1] queue.toKey('')
13
+ ARGV[2] jobId
14
+
15
+ Events:
16
+ 'waiting'
17
+ ]]
18
+ local rcall = redis.call;
19
+ local jobId = ARGV[2]
20
+
21
+ -- Includes
22
+ --- @include "includes/getTargetQueueList"
23
+
24
+ if rcall("ZREM", KEYS[1], jobId) == 1 then
25
+ local priority = tonumber(rcall("HGET", ARGV[1] .. jobId, "priority")) or 0
26
+ local target = getTargetQueueList(KEYS[4], KEYS[2], KEYS[3])
27
+
28
+ -- Remove delayed "marker" from the wait list if there is any.
29
+ -- Since we are adding a job we do not need the marker anymore.
30
+ if rcall("LINDEX", target, 0) == "0" then
31
+ rcall("LPOP", target)
32
+ end
33
+
34
+ if priority == 0 then
35
+ -- LIFO or FIFO
36
+ rcall("LPUSH", target, jobId)
37
+ else
38
+ -- Priority add
39
+ rcall("ZADD", KEYS[5], priority, jobId)
40
+ local count = rcall("ZCOUNT", KEYS[5], 0, priority)
41
+
42
+ local len = rcall("LLEN", target)
43
+ local id = rcall("LINDEX", target, len - (count - 1))
44
+ if id then
45
+ rcall("LINSERT", target, "BEFORE", id, jobId)
46
+ else
47
+ rcall("RPUSH", target, jobId)
48
+ end
49
+ end
50
+
51
+ -- Emit waiting event (wait..ing@token)
52
+ rcall("XADD", KEYS[6], "*", "event", "waiting", "jobId", jobId, "prev", "delayed");
53
+
54
+ rcall("HSET", ARGV[1] .. jobId, "delay", 0)
55
+
56
+ return 0
57
+ else
58
+ return -3
59
+ end
@@ -0,0 +1,19 @@
1
+ --[[
2
+ Release lock
3
+
4
+ Input:
5
+ KEYS[1] 'lock',
6
+
7
+ ARGV[1] token
8
+ ARGV[2] lock duration in milliseconds
9
+
10
+ Output:
11
+ "OK" if lock extented succesfully.
12
+ ]]
13
+ local rcall = redis.call
14
+
15
+ if rcall("GET", KEYS[1]) == ARGV[1] then
16
+ return rcall("DEL", KEYS[1])
17
+ else
18
+ return 0
19
+ end
@@ -0,0 +1,72 @@
1
+ --[[
2
+ Remove a job from all the queues it may be in as well as all its data.
3
+ In order to be able to remove a job, it cannot be active.
4
+
5
+ Input:
6
+ KEYS[1] queue prefix
7
+ ARGV[1] jobId
8
+
9
+ Events:
10
+ 'removed'
11
+ ]]
12
+
13
+ local rcall = redis.call
14
+
15
+ -- Includes
16
+ --- @include "includes/destructureJobKey"
17
+ --- @include "includes/isLocked"
18
+ --- @include "includes/removeJobFromAnyState"
19
+ --- @include "includes/removeParentDependencyKey"
20
+
21
+ local function removeJob( prefix, jobId, parentKey)
22
+ local jobKey = prefix .. jobId;
23
+
24
+ removeParentDependencyKey(jobKey, false, parentKey)
25
+
26
+ -- Check if this job has children
27
+ -- If so, we are going to try to remove the children recursively in deep first way because
28
+ -- if some job is locked we must exit with and error.
29
+ --local countProcessed = rcall("HLEN", jobKey .. ":processed")
30
+ local processed = rcall("HGETALL", jobKey .. ":processed")
31
+
32
+ if (#processed > 0) then
33
+ for i = 1, #processed, 2 do
34
+ local childJobId = getJobIdFromKey(processed[i])
35
+ local childJobPrefix = getJobKeyPrefix(processed[i], childJobId)
36
+ removeJob( childJobPrefix, childJobId, jobKey )
37
+ end
38
+ end
39
+
40
+ local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
41
+ if (#dependencies > 0) then
42
+ for i, childJobKey in ipairs(dependencies) do
43
+ -- We need to get the jobId for this job.
44
+ local childJobId = getJobIdFromKey(childJobKey)
45
+ local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
46
+ removeJob( childJobPrefix, childJobId, jobKey )
47
+ end
48
+ end
49
+
50
+ local prev = removeJobFromAnyState(prefix, jobId)
51
+
52
+ rcall("ZREM", prefix .. "priority", jobId)
53
+ rcall("DEL", jobKey, jobKey .. ":logs", jobKey .. ":dependencies", jobKey .. ":processed")
54
+
55
+ -- -- delete keys related to rate limiter
56
+ -- local limiterIndexTable = KEYS[10] .. ":index"
57
+ -- local limitedSetKey = rcall("HGET", limiterIndexTable, jobId)
58
+ -- if limitedSetKey then
59
+ -- rcall("SREM", limitedSetKey, jobId)
60
+ -- rcall("HDEL", limiterIndexTable, jobId)
61
+ -- end
62
+
63
+ rcall("XADD", prefix .. "events", "*", "event", "removed", "jobId", jobId, "prev", prev);
64
+ end
65
+
66
+ local prefix = KEYS[1]
67
+
68
+ if not isLocked(prefix, ARGV[1]) then
69
+ removeJob(prefix, ARGV[1])
70
+ return 1
71
+ end
72
+ return 0
@@ -0,0 +1,32 @@
1
+
2
+ --[[
3
+ Removes a repeatable job
4
+ Input:
5
+ KEYS[1] repeat jobs key
6
+ KEYS[2] delayed jobs key
7
+
8
+ ARGV[1] repeat job id
9
+ ARGV[2] repeat job key
10
+ ARGV[3] queue key
11
+
12
+ Output:
13
+ 0 - OK
14
+ 1 - Missing repeat job
15
+ ]]
16
+ local rcall = redis.call
17
+ local millis = rcall("ZSCORE", KEYS[1], ARGV[2])
18
+
19
+ if(millis) then
20
+ -- Delete next programmed job.
21
+ local repeatJobId = ARGV[1] .. millis
22
+ if(rcall("ZREM", KEYS[2], repeatJobId) == 1) then
23
+ rcall("DEL", ARGV[3] .. repeatJobId)
24
+ rcall("XADD", ARGV[3] .. "events", "*", "event", "removed", "jobId", repeatJobId, "prev", "delayed");
25
+ end
26
+ end
27
+
28
+ if(rcall("ZREM", KEYS[1], ARGV[2]) == 1) then
29
+ return 0
30
+ end
31
+
32
+ return 1
@@ -0,0 +1,35 @@
1
+ --[[
2
+ Attempts to reprocess a job
3
+
4
+ Input:
5
+ KEYS[1] job key
6
+ KEYS[2] events stream
7
+ KEYS[3] job state
8
+ KEYS[4] wait key
9
+
10
+ ARGV[1] job.id
11
+ ARGV[2] (job.opts.lifo ? 'R' : 'L') + 'PUSH'
12
+ ARGV[3] propVal - failedReason/returnvalue
13
+ ARGV[4] prev state - failed/completed
14
+
15
+ Output:
16
+ 1 means the operation was a success
17
+ -1 means the job does not exist
18
+ -3 means the job was not found in the expected set.
19
+ ]]
20
+ local rcall = redis.call;
21
+ if (rcall("EXISTS", KEYS[1]) == 1) then
22
+ local jobId = ARGV[1]
23
+ if (rcall("ZREM", KEYS[3], jobId) == 1) then
24
+ rcall(ARGV[2], KEYS[4], jobId)
25
+ rcall("HDEL", KEYS[1], "finishedOn", "processedOn", ARGV[3])
26
+
27
+ -- Emit waiting event
28
+ rcall("XADD", KEYS[2], "*", "event", "waiting", "jobId", jobId, "prev", ARGV[4]);
29
+ return 1
30
+ else
31
+ return -3
32
+ end
33
+ else
34
+ return -1
35
+ end
@@ -0,0 +1,51 @@
1
+ --[[
2
+ Retries a failed job by moving it back to the wait queue.
3
+
4
+ Input:
5
+ KEYS[1] 'active',
6
+ KEYS[2] 'wait'
7
+ KEYS[3] 'paused'
8
+ KEYS[4] job key
9
+ KEYS[5] 'meta'
10
+ KEYS[6] events stream
11
+
12
+ ARGV[1] pushCmd
13
+ ARGV[2] jobId
14
+ ARGV[3] token
15
+
16
+ Events:
17
+ 'waiting'
18
+
19
+ Output:
20
+ 0 - OK
21
+ -1 - Missing key
22
+ -2 - Missing lock
23
+ ]]
24
+ local rcall = redis.call
25
+
26
+ -- Includes
27
+ --- @include "includes/getTargetQueueList"
28
+
29
+ if rcall("EXISTS", KEYS[4]) == 1 then
30
+
31
+ if ARGV[3] ~= "0" then
32
+ local lockKey = KEYS[4] .. ':lock'
33
+ if rcall("GET", lockKey) == ARGV[3] then
34
+ rcall("DEL", lockKey)
35
+ else
36
+ return -2
37
+ end
38
+ end
39
+
40
+ local target = getTargetQueueList(KEYS[5], KEYS[2], KEYS[3])
41
+
42
+ rcall("LREM", KEYS[1], 0, ARGV[2])
43
+ rcall(ARGV[1], target, ARGV[2])
44
+
45
+ -- Emit waiting event
46
+ rcall("XADD", KEYS[6], "*", "event", "waiting", "jobId", ARGV[2], "prev", "failed");
47
+
48
+ return 0
49
+ else
50
+ return -1
51
+ end