bullmq 2.3.1 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (117) hide show
  1. package/dist/cjs/classes/flow-producer.js +16 -1
  2. package/dist/cjs/classes/flow-producer.js.map +1 -1
  3. package/dist/cjs/classes/redis-connection.js +1 -1
  4. package/dist/cjs/classes/redis-connection.js.map +1 -1
  5. package/dist/cjs/commands/addJob-8.lua +174 -0
  6. package/dist/cjs/commands/changeDelay-4.lua +43 -0
  7. package/dist/cjs/commands/cleanJobsInSet-2.lua +46 -0
  8. package/dist/cjs/commands/drain-4.lua +25 -0
  9. package/dist/cjs/commands/extendLock-2.lua +23 -0
  10. package/dist/cjs/commands/getState-7.lua +57 -0
  11. package/dist/cjs/commands/getStateV2-7.lua +51 -0
  12. package/dist/cjs/commands/includes/addJobWithPriority.lua +16 -0
  13. package/dist/cjs/commands/includes/batches.lua +18 -0
  14. package/dist/cjs/commands/includes/checkItemInList.lua +12 -0
  15. package/dist/cjs/commands/includes/checkStalledJobs.lua +137 -0
  16. package/dist/cjs/commands/includes/cleanList.lua +50 -0
  17. package/dist/cjs/commands/includes/cleanSet.lua +50 -0
  18. package/dist/cjs/commands/includes/collectMetrics.lua +46 -0
  19. package/dist/cjs/commands/includes/destructureJobKey.lua +12 -0
  20. package/dist/cjs/commands/includes/getNextDelayedTimestamp.lua +13 -0
  21. package/dist/cjs/commands/includes/getTargetQueueList.lua +12 -0
  22. package/dist/cjs/commands/includes/getTimestamp.lua +19 -0
  23. package/dist/cjs/commands/includes/getZSetItems.lua +7 -0
  24. package/dist/cjs/commands/includes/isLocked.lua +31 -0
  25. package/dist/cjs/commands/includes/moveJobFromWaitToActive.lua +84 -0
  26. package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +28 -0
  27. package/dist/cjs/commands/includes/promoteDelayedJobs.lua +49 -0
  28. package/dist/cjs/commands/includes/removeJob.lua +13 -0
  29. package/dist/cjs/commands/includes/removeJobFromAnyState.lua +26 -0
  30. package/dist/cjs/commands/includes/removeJobs.lua +38 -0
  31. package/dist/cjs/commands/includes/removeJobsByMaxAge.lua +15 -0
  32. package/dist/cjs/commands/includes/removeJobsByMaxCount.lua +15 -0
  33. package/dist/cjs/commands/includes/removeParentDependencyKey.lua +81 -0
  34. package/dist/cjs/commands/includes/trimEvents.lua +12 -0
  35. package/dist/cjs/commands/includes/updateParentDepsIfNeeded.lua +28 -0
  36. package/dist/cjs/commands/isFinished-3.lua +48 -0
  37. package/dist/cjs/commands/isJobInList-1.lua +16 -0
  38. package/dist/cjs/commands/moveStalledJobsToWait-8.lua +24 -0
  39. package/dist/cjs/commands/moveToActive-9.lua +67 -0
  40. package/dist/cjs/commands/moveToDelayed-5.lua +54 -0
  41. package/dist/cjs/commands/moveToFinished-12.lua +201 -0
  42. package/dist/cjs/commands/moveToWaitingChildren-4.lua +62 -0
  43. package/dist/cjs/commands/obliterate-2.lua +94 -0
  44. package/dist/cjs/commands/pause-4.lua +27 -0
  45. package/dist/cjs/commands/promote-6.lua +51 -0
  46. package/dist/cjs/commands/releaseLock-1.lua +19 -0
  47. package/dist/cjs/commands/removeJob-1.lua +72 -0
  48. package/dist/cjs/commands/removeRepeatable-2.lua +32 -0
  49. package/dist/cjs/commands/reprocessJob-4.lua +35 -0
  50. package/dist/cjs/commands/retryJob-6.lua +51 -0
  51. package/dist/cjs/commands/retryJobs-6.lua +53 -0
  52. package/dist/cjs/commands/takeLock-1.lua +17 -0
  53. package/dist/cjs/commands/updateData-1.lua +16 -0
  54. package/dist/cjs/commands/updateProgress-2.lua +22 -0
  55. package/dist/cjs/scripts/moveToFinished-12.js +5 -2
  56. package/dist/cjs/scripts/moveToFinished-12.js.map +1 -1
  57. package/dist/cjs/scripts/promote-6.js +15 -9
  58. package/dist/cjs/scripts/promote-6.js.map +1 -1
  59. package/dist/esm/classes/flow-producer.js +16 -1
  60. package/dist/esm/classes/flow-producer.js.map +1 -1
  61. package/dist/esm/classes/redis-connection.js +1 -1
  62. package/dist/esm/classes/redis-connection.js.map +1 -1
  63. package/dist/esm/commands/addJob-8.lua +174 -0
  64. package/dist/esm/commands/changeDelay-4.lua +43 -0
  65. package/dist/esm/commands/cleanJobsInSet-2.lua +46 -0
  66. package/dist/esm/commands/drain-4.lua +25 -0
  67. package/dist/esm/commands/extendLock-2.lua +23 -0
  68. package/dist/esm/commands/getState-7.lua +57 -0
  69. package/dist/esm/commands/getStateV2-7.lua +51 -0
  70. package/dist/esm/commands/includes/addJobWithPriority.lua +16 -0
  71. package/dist/esm/commands/includes/batches.lua +18 -0
  72. package/dist/esm/commands/includes/checkItemInList.lua +12 -0
  73. package/dist/esm/commands/includes/checkStalledJobs.lua +137 -0
  74. package/dist/esm/commands/includes/cleanList.lua +50 -0
  75. package/dist/esm/commands/includes/cleanSet.lua +50 -0
  76. package/dist/esm/commands/includes/collectMetrics.lua +46 -0
  77. package/dist/esm/commands/includes/destructureJobKey.lua +12 -0
  78. package/dist/esm/commands/includes/getNextDelayedTimestamp.lua +13 -0
  79. package/dist/esm/commands/includes/getTargetQueueList.lua +12 -0
  80. package/dist/esm/commands/includes/getTimestamp.lua +19 -0
  81. package/dist/esm/commands/includes/getZSetItems.lua +7 -0
  82. package/dist/esm/commands/includes/isLocked.lua +31 -0
  83. package/dist/esm/commands/includes/moveJobFromWaitToActive.lua +84 -0
  84. package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +28 -0
  85. package/dist/esm/commands/includes/promoteDelayedJobs.lua +49 -0
  86. package/dist/esm/commands/includes/removeJob.lua +13 -0
  87. package/dist/esm/commands/includes/removeJobFromAnyState.lua +26 -0
  88. package/dist/esm/commands/includes/removeJobs.lua +38 -0
  89. package/dist/esm/commands/includes/removeJobsByMaxAge.lua +15 -0
  90. package/dist/esm/commands/includes/removeJobsByMaxCount.lua +15 -0
  91. package/dist/esm/commands/includes/removeParentDependencyKey.lua +81 -0
  92. package/dist/esm/commands/includes/trimEvents.lua +12 -0
  93. package/dist/esm/commands/includes/updateParentDepsIfNeeded.lua +28 -0
  94. package/dist/esm/commands/isFinished-3.lua +48 -0
  95. package/dist/esm/commands/isJobInList-1.lua +16 -0
  96. package/dist/esm/commands/moveStalledJobsToWait-8.lua +24 -0
  97. package/dist/esm/commands/moveToActive-9.lua +67 -0
  98. package/dist/esm/commands/moveToDelayed-5.lua +54 -0
  99. package/dist/esm/commands/moveToFinished-12.lua +201 -0
  100. package/dist/esm/commands/moveToWaitingChildren-4.lua +62 -0
  101. package/dist/esm/commands/obliterate-2.lua +94 -0
  102. package/dist/esm/commands/pause-4.lua +27 -0
  103. package/dist/esm/commands/promote-6.lua +51 -0
  104. package/dist/esm/commands/releaseLock-1.lua +19 -0
  105. package/dist/esm/commands/removeJob-1.lua +72 -0
  106. package/dist/esm/commands/removeRepeatable-2.lua +32 -0
  107. package/dist/esm/commands/reprocessJob-4.lua +35 -0
  108. package/dist/esm/commands/retryJob-6.lua +51 -0
  109. package/dist/esm/commands/retryJobs-6.lua +53 -0
  110. package/dist/esm/commands/takeLock-1.lua +17 -0
  111. package/dist/esm/commands/updateData-1.lua +16 -0
  112. package/dist/esm/commands/updateProgress-2.lua +22 -0
  113. package/dist/esm/scripts/moveToFinished-12.js +5 -2
  114. package/dist/esm/scripts/moveToFinished-12.js.map +1 -1
  115. package/dist/esm/scripts/promote-6.js +15 -9
  116. package/dist/esm/scripts/promote-6.js.map +1 -1
  117. package/package.json +2 -2
@@ -0,0 +1,81 @@
1
+ --[[
2
+ Check if this job has a parent. If so we will just remove it from
3
+ the parent child list, but if it is the last child we should move the parent to "wait/paused"
4
+ which requires code from "moveToFinished"
5
+ ]]
6
+
7
+ --- @include "destructureJobKey"
8
+ --- @include "getTargetQueueList"
9
+
10
+ local function moveParentToWait(parentPrefix, parentId, emitEvent)
11
+ local parentTarget = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "wait", parentPrefix .. "paused")
12
+ rcall("RPUSH", parentTarget, parentId)
13
+
14
+ if emitEvent then
15
+ local parentEventStream = parentPrefix .. "events"
16
+ rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
17
+ end
18
+ end
19
+
20
+ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
21
+ if parentKey then
22
+ local parentProcessedKey = parentKey .. ":processed"
23
+ rcall("HDEL", parentProcessedKey, jobKey)
24
+ local parentDependenciesKey = parentKey .. ":dependencies"
25
+ local result = rcall("SREM", parentDependenciesKey, jobKey)
26
+ if result > 0 then
27
+ local pendingDependencies = rcall("SCARD", parentDependenciesKey)
28
+ if pendingDependencies == 0 then
29
+ local parentId = getJobIdFromKey(parentKey)
30
+ local parentPrefix = getJobKeyPrefix(parentKey, parentId)
31
+
32
+ local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
33
+
34
+ if numRemovedElements == 1 then
35
+ if hard then
36
+ if parentPrefix == baseKey then
37
+ removeParentDependencyKey(parentKey, hard, nil, baseKey)
38
+ rcall("DEL", parentKey, parentKey .. ':logs',
39
+ parentKey .. ':dependencies', parentKey .. ':processed')
40
+ else
41
+ moveParentToWait(parentPrefix, parentId)
42
+ end
43
+ else
44
+ moveParentToWait(parentPrefix, parentId, true)
45
+ end
46
+ end
47
+ end
48
+ end
49
+ else
50
+ local missedParentKey = rcall("HGET", jobKey, "parentKey")
51
+ if( (type(missedParentKey) == "string") and missedParentKey ~= "" and (rcall("EXISTS", missedParentKey) == 1)) then
52
+ local parentProcessedKey = missedParentKey .. ":processed"
53
+ rcall("HDEL", parentProcessedKey, jobKey)
54
+ local parentDependenciesKey = missedParentKey .. ":dependencies"
55
+ local result = rcall("SREM", parentDependenciesKey, jobKey)
56
+ if result > 0 then
57
+ local pendingDependencies = rcall("SCARD", parentDependenciesKey)
58
+ if pendingDependencies == 0 then
59
+ local parentId = getJobIdFromKey(missedParentKey)
60
+ local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
61
+
62
+ local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
63
+
64
+ if numRemovedElements == 1 then
65
+ if hard then
66
+ if parentPrefix == baseKey then
67
+ removeParentDependencyKey(missedParentKey, hard, nil, baseKey)
68
+ rcall("DEL", missedParentKey, missedParentKey .. ':logs',
69
+ missedParentKey .. ':dependencies', missedParentKey .. ':processed')
70
+ else
71
+ moveParentToWait(parentPrefix, parentId)
72
+ end
73
+ else
74
+ moveParentToWait(parentPrefix, parentId, true)
75
+ end
76
+ end
77
+ end
78
+ end
79
+ end
80
+ end
81
+ end
@@ -0,0 +1,12 @@
1
+ --[[
2
+ Function to trim events, default 10000.
3
+ ]]
4
+
5
+ local function trimEvents(metaKey, eventStreamKey)
6
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
7
+ if maxEvents ~= false then
8
+ rcall("XTRIM", eventStreamKey, "MAXLEN", "~", maxEvents)
9
+ else
10
+ rcall("XTRIM", eventStreamKey, "MAXLEN", "~", 10000)
11
+ end
12
+ end
@@ -0,0 +1,28 @@
1
+ --[[
2
+ Validate and move or add dependencies to parent.
3
+ ]]
4
+
5
+ -- Includes
6
+ --- @include "addJobWithPriority"
7
+ --- @include "getTargetQueueList"
8
+
9
+ local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
10
+ parentId, jobIdKey, returnvalue )
11
+ local processedSet = parentKey .. ":processed"
12
+ rcall("HSET", processedSet, jobIdKey, returnvalue)
13
+ local activeParent = rcall("ZSCORE", parentQueueKey .. ":waiting-children", parentId)
14
+ if rcall("SCARD", parentDependenciesKey) == 0 and activeParent then
15
+ rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
16
+ local parentTarget = getTargetQueueList(parentQueueKey .. ":meta", parentQueueKey .. ":wait",
17
+ parentQueueKey .. ":paused")
18
+ local priority = tonumber(rcall("HGET", parentKey, "priority"))
19
+ -- Standard or priority add
20
+ if priority == 0 then
21
+ rcall("RPUSH", parentTarget, parentId)
22
+ else
23
+ addJobWithPriority(parentQueueKey .. ":priority", priority, parentTarget, parentId)
24
+ end
25
+
26
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
27
+ end
28
+ end
@@ -0,0 +1,48 @@
1
+ --[[
2
+ Checks if a job is finished (.i.e. is in the completed or failed set)
3
+
4
+ Input:
5
+ KEYS[1] completed key
6
+ KEYS[2] failed key
7
+ KEYS[3] job key
8
+
9
+ ARGV[1] job id
10
+ ARGV[2] return value?
11
+ Output:
12
+ 0 - Not finished.
13
+ 1 - Completed.
14
+ 2 - Failed.
15
+ -5 - Missing job.
16
+ ]]
17
+ local rcall = redis.call
18
+ if rcall("EXISTS", KEYS[3]) ~= 1 then
19
+ if ARGV[2] == "1" then
20
+
21
+ return {-5,"Missing key for job " .. KEYS[3] .. ". isFinished"}
22
+ end
23
+ return -5
24
+ end
25
+
26
+ if rcall("ZSCORE", KEYS[1], ARGV[1]) ~= false then
27
+ if ARGV[2] == "1" then
28
+ local returnValue = rcall("HGET", KEYS[3], "returnvalue")
29
+
30
+ return {1,returnValue}
31
+ end
32
+ return 1
33
+ end
34
+
35
+ if rcall("ZSCORE", KEYS[2], ARGV[1]) ~= false then
36
+ if ARGV[2] == "1" then
37
+ local failedReason = rcall("HGET", KEYS[3], "failedReason")
38
+
39
+ return {2,failedReason}
40
+ end
41
+ return 2
42
+ end
43
+
44
+ if ARGV[2] == "1" then
45
+ return {0}
46
+ end
47
+
48
+ return 0
@@ -0,0 +1,16 @@
1
+ --[[
2
+ Checks if job is in a given list.
3
+
4
+ Input:
5
+ KEYS[1]
6
+ ARGV[1]
7
+
8
+ Output:
9
+ 1 if element found in the list.
10
+ ]]
11
+
12
+ -- Includes
13
+ --- @include "includes/checkItemInList"
14
+
15
+ local items = redis.call("LRANGE", KEYS[1] , 0, -1)
16
+ return checkItemInList(items, ARGV[1])
@@ -0,0 +1,24 @@
1
+ --[[
2
+ Move stalled jobs to wait.
3
+
4
+ Input:
5
+ KEYS[1] 'stalled' (SET)
6
+ KEYS[2] 'wait', (LIST)
7
+ KEYS[3] 'active', (LIST)
8
+ KEYS[4] 'failed', (ZSET)
9
+ KEYS[5] 'stalled-check', (KEY)
10
+ KEYS[6] 'meta', (KEY)
11
+ KEYS[7] 'paused', (LIST)
12
+ KEYS[8] 'event stream' (STREAM)
13
+
14
+ ARGV[1] Max stalled job count
15
+ ARGV[2] queue.toKey('')
16
+ ARGV[3] timestamp
17
+ ARGV[4] max check time
18
+
19
+ Events:
20
+ 'stalled' with stalled job id.
21
+ ]] -- Includes
22
+ --- @include "includes/checkStalledJobs"
23
+ return checkStalledJobs(KEYS[1], KEYS[2], KEYS[3], KEYS[4], KEYS[5], KEYS[6],
24
+ KEYS[7], KEYS[8], ARGV[1], ARGV[2], ARGV[3], ARGV[4])
@@ -0,0 +1,67 @@
1
+ --[[
2
+ Move next job to be processed to active, lock it and fetch its data. The job
3
+ may be delayed, in that case we need to move it to the delayed set instead.
4
+
5
+ This operation guarantees that the worker owns the job during the lock
6
+ expiration time. The worker is responsible of keeping the lock fresh
7
+ so that no other worker picks this job again.
8
+
9
+ Input:
10
+ KEYS[1] wait key
11
+ KEYS[2] active key
12
+ KEYS[3] priority key
13
+ KEYS[4] stream events key
14
+ KEYS[5] stalled key
15
+
16
+ -- Rate limiting
17
+ KEYS[6] rate limiter key
18
+ KEYS[7] delayed key
19
+
20
+ -- Promote delayed jobs
21
+ KEYS[8] paused key
22
+ KEYS[9] meta key
23
+
24
+ -- Arguments
25
+ ARGV[1] key prefix
26
+ ARGV[2] timestamp
27
+ ARGV[3] optional job ID
28
+ ARGV[4] opts
29
+
30
+ opts - token - lock token
31
+ opts - lockDuration
32
+ opts - limiter
33
+ ]]
34
+ local jobId
35
+ local rcall = redis.call
36
+
37
+ -- Includes
38
+ --- @include "includes/moveJobFromWaitToActive"
39
+ --- @include "includes/getNextDelayedTimestamp"
40
+ --- @include "includes/promoteDelayedJobs"
41
+
42
+ -- Check if there are delayed jobs that we can move to wait.
43
+ promoteDelayedJobs(KEYS[7], KEYS[1], KEYS[3], KEYS[8], KEYS[9], KEYS[4], ARGV[1], ARGV[2])
44
+
45
+ if (ARGV[3] ~= "") then
46
+ jobId = ARGV[3]
47
+ -- clean stalled key
48
+ rcall("SREM", KEYS[5], jobId)
49
+ else
50
+ -- no job ID, try non-blocking move from wait to active
51
+ jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
52
+ end
53
+
54
+ -- If jobId is special ID 0, then there is no job to process
55
+ if jobId == "0" then
56
+ rcall("LREM", KEYS[2], 1, 0)
57
+ elseif jobId then
58
+ local opts = cmsgpack.unpack(ARGV[4])
59
+ -- this script is not really moving, it is preparing the job for processing
60
+ return moveJobFromWaitToActive(KEYS, ARGV[1], jobId, ARGV[2], opts)
61
+ end
62
+
63
+ -- Return the timestamp for the next delayed job if any.
64
+ local nextTimestamp = getNextDelayedTimestamp(KEYS[7])
65
+ if (nextTimestamp ~= nil) then
66
+ return nextTimestamp - tonumber(ARGV[2])
67
+ end
@@ -0,0 +1,54 @@
1
+ --[[
2
+ Moves job from active to delayed set.
3
+
4
+ Input:
5
+ KEYS[1] active key
6
+ KEYS[2] delayed key
7
+ KEYS[3] job key
8
+ KEYS[4] events stream
9
+ KEYS[5] delayed stream
10
+
11
+ ARGV[1] delayedTimestamp
12
+ ARGV[2] the id of the job
13
+ ARGV[3] queue token
14
+
15
+ Output:
16
+ 0 - OK
17
+ -1 - Missing job.
18
+ -3 - Job not in active set.
19
+
20
+ Events:
21
+ - delayed key.
22
+ ]]
23
+ local rcall = redis.call
24
+
25
+ if rcall("EXISTS", KEYS[3]) == 1 then
26
+
27
+ if ARGV[3] ~= "0" then
28
+ local lockKey = KEYS[3] .. ':lock'
29
+ if rcall("GET", lockKey) == ARGV[3] then
30
+ rcall("DEL", lockKey)
31
+ else
32
+ return -2
33
+ end
34
+ end
35
+
36
+ local jobId = ARGV[2]
37
+ local score = tonumber(ARGV[1])
38
+ local delayedTimestamp = (score / 0x1000)
39
+
40
+ local numRemovedElements = rcall("LREM", KEYS[1], -1, jobId)
41
+
42
+ if(numRemovedElements < 1) then
43
+ return -3
44
+ end
45
+
46
+ rcall("ZADD", KEYS[2], score, jobId)
47
+
48
+ rcall("XADD", KEYS[4], "*", "event", "delayed", "jobId", jobId, "delay", delayedTimestamp);
49
+ rcall("XADD", KEYS[5], "*", "nextTimestamp", delayedTimestamp);
50
+
51
+ return 0
52
+ else
53
+ return -1
54
+ end
@@ -0,0 +1,201 @@
1
+ --[[
2
+ Move job from active to a finished status (completed o failed)
3
+ A job can only be moved to completed if it was active.
4
+ The job must be locked before it can be moved to a finished status,
5
+ and the lock must be released in this script.
6
+
7
+ Input:
8
+ KEYS[1] wait key
9
+ KEYS[2] active key
10
+ KEYS[3] priority key
11
+ KEYS[4] event stream key
12
+ KEYS[5] stalled key
13
+
14
+ -- Rate limiting
15
+ KEYS[6] rate limiter key
16
+ KEYS[7] delayed key
17
+
18
+ KEYS[8] paused key
19
+
20
+ KEYS[9] completed/failed key
21
+ KEYS[10] jobId key
22
+ KEYS[11] meta key
23
+ KEYS[12] metrics key
24
+
25
+ ARGV[1] jobId
26
+ ARGV[2] timestamp
27
+ ARGV[3] msg property returnvalue / failedReason
28
+ ARGV[4] return value / failed reason
29
+ ARGV[5] target (completed/failed)
30
+ ARGV[6] event data (? maybe just send jobid).
31
+ ARGV[7] fetch next?
32
+ ARGV[8] keys prefix
33
+ ARGV[9] opts
34
+
35
+ opts - token - lock token
36
+ opts - keepJobs
37
+ opts - lockDuration - lock duration in milliseconds
38
+ opts - parent - parent data
39
+ opts - parentKey
40
+ opts - attempts max attempts
41
+ opts - attemptsMade
42
+ opts - maxMetricsSize
43
+ opts - fpof - fail parent on fail
44
+
45
+ Output:
46
+ 0 OK
47
+ -1 Missing key.
48
+ -2 Missing lock.
49
+ -3 Job not in active set
50
+ -4 Job has pending dependencies
51
+
52
+ Events:
53
+ 'completed/failed'
54
+ ]]
55
+ local rcall = redis.call
56
+
57
+ --- Includes
58
+ --- @include "includes/collectMetrics"
59
+ --- @include "includes/getNextDelayedTimestamp"
60
+ --- @include "includes/moveJobFromWaitToActive"
61
+ --- @include "includes/moveParentFromWaitingChildrenToFailed"
62
+ --- @include "includes/promoteDelayedJobs"
63
+ --- @include "includes/removeJobsByMaxAge"
64
+ --- @include "includes/removeJobsByMaxCount"
65
+ --- @include "includes/trimEvents"
66
+ --- @include "includes/updateParentDepsIfNeeded"
67
+
68
+ local jobIdKey = KEYS[10]
69
+ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
70
+ local opts = cmsgpack.unpack(ARGV[9])
71
+
72
+ local token = opts['token']
73
+ local parentId = opts['parent'] and opts['parent']['id'] or ""
74
+ local parentQueueKey = opts['parent'] and opts['parent']['queue'] or ""
75
+ local parentKey = opts['parentKey'] or ""
76
+ local attempts = opts['attempts']
77
+ local attemptsMade = opts['attemptsMade']
78
+ local maxMetricsSize = opts['maxMetricsSize']
79
+ local maxCount = opts['keepJobs']['count']
80
+ local maxAge = opts['keepJobs']['age']
81
+
82
+ if token ~= "0" then
83
+ local lockKey = jobIdKey .. ':lock'
84
+ if rcall("GET", lockKey) == token then
85
+ rcall("DEL", lockKey)
86
+ rcall("SREM", KEYS[5], ARGV[1])
87
+ else
88
+ return -2
89
+ end
90
+ end
91
+
92
+ if rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then -- // Make sure it does not have pending dependencies
93
+ return -4
94
+ end
95
+
96
+ local jobId = ARGV[1]
97
+ local timestamp = ARGV[2]
98
+
99
+ -- Remove from active list (if not active we shall return error)
100
+ local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
101
+
102
+ if (numRemovedElements < 1) then return -3 end
103
+
104
+ -- Trim events before emiting them to avoid trimming events emitted in this script
105
+ trimEvents(KEYS[11], KEYS[4])
106
+
107
+ -- If job has a parent we need to
108
+ -- 1) remove this job id from parents dependencies
109
+ -- 2) move the job Id to parent "processed" set
110
+ -- 3) push the results into parent "results" list
111
+ -- 4) if parent's dependencies is empty, then move parent to "wait/paused". Note it may be a different queue!.
112
+ if parentId == "" and parentKey ~= "" then
113
+ parentId = getJobIdFromKey(parentKey)
114
+ parentQueueKey = getJobKeyPrefix(parentKey, ":" .. parentId)
115
+ end
116
+ if parentId ~= "" then
117
+ if ARGV[5] == "completed" then
118
+ local dependenciesSet = parentKey .. ":dependencies"
119
+ if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
120
+ updateParentDepsIfNeeded(parentKey, parentQueueKey, dependenciesSet,
121
+ parentId, jobIdKey, ARGV[4])
122
+ end
123
+ elseif opts['fpof'] then
124
+ moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
125
+ end
126
+ end
127
+
128
+ -- Remove job?
129
+ if maxCount ~= 0 then
130
+ local targetSet = KEYS[9]
131
+ -- Add to complete/failed set
132
+ rcall("ZADD", targetSet, timestamp, jobId)
133
+ rcall("HMSET", jobIdKey, ARGV[3], ARGV[4], "finishedOn", timestamp)
134
+ -- "returnvalue" / "failedReason" and "finishedOn"
135
+
136
+ -- Remove old jobs?
137
+ local prefix = ARGV[8]
138
+
139
+ if maxAge ~= nil then
140
+ removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
141
+ end
142
+
143
+ if maxCount ~= nil and maxCount > 0 then
144
+ removeJobsByMaxCount(maxCount, targetSet, prefix)
145
+ end
146
+ else
147
+ rcall("DEL", jobIdKey, jobIdKey .. ':logs', jobIdKey .. ':processed')
148
+ end
149
+
150
+ rcall("XADD", KEYS[4], "*", "event", ARGV[5], "jobId", jobId, ARGV[3],
151
+ ARGV[4])
152
+
153
+ if ARGV[5] == "failed" then
154
+ if tonumber(attemptsMade) >= tonumber(attempts) then
155
+ rcall("XADD", KEYS[4], "*", "event", "retries-exhausted", "jobId",
156
+ jobId, "attemptsMade", attemptsMade)
157
+ end
158
+ end
159
+
160
+ -- Collect metrics
161
+ if maxMetricsSize ~= "" then
162
+ collectMetrics(KEYS[12], KEYS[12]..':data', maxMetricsSize, timestamp)
163
+ end
164
+
165
+ -- Try to get next job to avoid an extra roundtrip if the queue is not closing,
166
+ -- and not rate limited.
167
+ if (ARGV[7] == "1") then
168
+
169
+ -- Check if there are delayed jobs that can be promoted
170
+ promoteDelayedJobs(KEYS[7], KEYS[1], KEYS[3], KEYS[8], KEYS[11], KEYS[4], ARGV[8], timestamp)
171
+
172
+ jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
173
+
174
+ if jobId == "0" then
175
+ rcall("LREM", KEYS[2], 1, 0)
176
+ elseif jobId then
177
+ return moveJobFromWaitToActive(KEYS, ARGV[8], jobId, timestamp, opts)
178
+ end
179
+
180
+ -- Return the timestamp for the next delayed job if any.
181
+ local nextTimestamp = getNextDelayedTimestamp(KEYS[7])
182
+ if (nextTimestamp ~= nil) then
183
+ -- The result is guaranteed to be positive, since the
184
+ -- ZRANGEBYSCORE command would have return a job otherwise.
185
+ return nextTimestamp - timestamp
186
+ end
187
+ end
188
+
189
+ local waitLen = rcall("LLEN", KEYS[1])
190
+ if waitLen == 0 then
191
+ local activeLen = rcall("LLEN", KEYS[2])
192
+
193
+ if activeLen == 0 then
194
+ rcall("XADD", KEYS[4], "*", "event", "drained")
195
+ end
196
+ end
197
+
198
+ return 0
199
+ else
200
+ return -1
201
+ end
@@ -0,0 +1,62 @@
1
+ --[[
2
+ Moves job from active to waiting children set.
3
+
4
+ Input:
5
+ KEYS[1] lock key
6
+ KEYS[2] active key
7
+ KEYS[3] waitChildrenKey key
8
+ KEYS[4] job key
9
+
10
+ ARGV[1] token
11
+ ARGV[2] child key
12
+ ARGV[3] timestamp
13
+ ARGV[4] the id of the job
14
+
15
+ Output:
16
+ 0 - OK
17
+ 1 - There are not pending dependencies.
18
+ -1 - Missing job.
19
+ -2 - Missing lock
20
+ -3 - Job not in active set
21
+ ]]
22
+ local rcall = redis.call
23
+
24
+ local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId, timestamp, lockKey, token)
25
+ if token ~= "0" then
26
+ if rcall("GET", lockKey) == token then
27
+ rcall("DEL", lockKey)
28
+ else
29
+ return -2
30
+ end
31
+ end
32
+
33
+ local score = tonumber(timestamp)
34
+
35
+ local numRemovedElements = rcall("LREM", activeKey, -1, jobId)
36
+
37
+ if(numRemovedElements < 1) then
38
+ return -3
39
+ end
40
+
41
+ rcall("ZADD", waitingChildrenKey, score, jobId)
42
+
43
+ return 0
44
+ end
45
+
46
+ if rcall("EXISTS", KEYS[4]) == 1 then
47
+ if ARGV[2] ~= "" then
48
+ if rcall("SISMEMBER", KEYS[4] .. ":dependencies", ARGV[2]) ~= 0 then
49
+ return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], ARGV[1])
50
+ end
51
+
52
+ return 1
53
+ else
54
+ if rcall("SCARD", KEYS[4] .. ":dependencies") ~= 0 then
55
+ return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], ARGV[1])
56
+ end
57
+
58
+ return 1
59
+ end
60
+ end
61
+
62
+ return -1
@@ -0,0 +1,94 @@
1
+ --[[
2
+ Completely obliterates a queue and all of its contents
3
+
4
+ Input:
5
+ KEYS[1] meta
6
+ KEYS[2] base
7
+
8
+ ARGV[1] count
9
+ ARGV[2] force
10
+ ]]
11
+
12
+ -- This command completely destroys a queue including all of its jobs, current or past
13
+ -- leaving no trace of its existence. Since this script needs to iterate to find all the job
14
+ -- keys, consider that this call may be slow for very large queues.
15
+
16
+ -- The queue needs to be "paused" or it will return an error
17
+ -- If the queue has currently active jobs then the script by default will return error,
18
+ -- however this behaviour can be overrided using the 'force' option.
19
+ local maxCount = tonumber(ARGV[1])
20
+ local baseKey = KEYS[2]
21
+
22
+ local rcall = redis.call
23
+
24
+ --- @include "includes/removeJobs"
25
+
26
+ local function removeLockKeys(keys)
27
+ for i, key in ipairs(keys) do
28
+ rcall("DEL", baseKey .. key .. ':lock')
29
+ end
30
+ end
31
+
32
+ -- 1) Check if paused, if not return with error.
33
+ if rcall("HEXISTS", KEYS[1], "paused") ~= 1 then
34
+ return -1 -- Error, NotPaused
35
+ end
36
+
37
+ -- 2) Check if there are active jobs, if there are and not "force" return error.
38
+ local activeKey = baseKey .. 'active'
39
+ local activeJobs = getListItems(activeKey, maxCount)
40
+ if (#activeJobs > 0) then
41
+ if(ARGV[2] == "") then
42
+ return -2 -- Error, ExistActiveJobs
43
+ end
44
+ end
45
+
46
+ removeLockKeys(activeJobs)
47
+ maxCount = removeJobs(activeJobs, true, baseKey, maxCount)
48
+ rcall("LTRIM", activeKey, #activeJobs, -1)
49
+ if(maxCount <= 0) then
50
+ return 1
51
+ end
52
+
53
+ local delayedKey = baseKey .. 'delayed'
54
+ maxCount = removeZSetJobs(delayedKey, true, baseKey, maxCount)
55
+ if(maxCount <= 0) then
56
+ return 1
57
+ end
58
+
59
+ local completedKey = baseKey .. 'completed'
60
+ maxCount = removeZSetJobs(completedKey, true, baseKey, maxCount)
61
+ if(maxCount <= 0) then
62
+ return 1
63
+ end
64
+
65
+ local waitKey = baseKey .. 'paused'
66
+ maxCount = removeListJobs(waitKey, true, baseKey, maxCount)
67
+ if(maxCount <= 0) then
68
+ return 1
69
+ end
70
+
71
+ local failedKey = baseKey .. 'failed'
72
+ maxCount = removeZSetJobs(failedKey, true, baseKey, maxCount)
73
+ if(maxCount <= 0) then
74
+ return 1
75
+ end
76
+
77
+ if(maxCount > 0) then
78
+ rcall("DEL",
79
+ baseKey .. 'priority',
80
+ baseKey .. 'events',
81
+ baseKey .. 'delay',
82
+ baseKey .. 'stalled-check',
83
+ baseKey .. 'stalled',
84
+ baseKey .. 'id',
85
+ baseKey .. 'meta',
86
+ baseKey .. 'repeat',
87
+ baseKey .. 'metrics:completed',
88
+ baseKey .. 'metrics:completed:data',
89
+ baseKey .. 'metrics:failed',
90
+ baseKey .. 'metrics:failed:data')
91
+ return 0
92
+ else
93
+ return 1
94
+ end