bullmq 5.58.9 → 5.60.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. package/dist/cjs/classes/child-pool.js +8 -1
  2. package/dist/cjs/classes/child-pool.js.map +1 -1
  3. package/dist/cjs/classes/queue.js +9 -0
  4. package/dist/cjs/classes/queue.js.map +1 -1
  5. package/dist/cjs/classes/scripts.js +4 -1
  6. package/dist/cjs/classes/scripts.js.map +1 -1
  7. package/dist/cjs/classes/worker.js +42 -30
  8. package/dist/cjs/classes/worker.js.map +1 -1
  9. package/dist/cjs/commands/changePriority-7.lua +1 -1
  10. package/dist/cjs/commands/{getRateLimitTtl-1.lua → getRateLimitTtl-2.lua} +6 -0
  11. package/dist/cjs/commands/includes/deduplicateJob.lua +53 -20
  12. package/dist/cjs/commands/includes/getTargetQueueList.lua +5 -5
  13. package/dist/cjs/commands/includes/prepareJobForProcessing.lua +2 -4
  14. package/dist/cjs/commands/moveJobFromActiveToWait-9.lua +1 -1
  15. package/dist/cjs/commands/moveStalledJobsToWait-8.lua +0 -1
  16. package/dist/cjs/commands/moveToActive-11.lua +7 -4
  17. package/dist/cjs/commands/moveToDelayed-8.lua +2 -1
  18. package/dist/cjs/commands/moveToFinished-14.lua +10 -7
  19. package/dist/cjs/scripts/addDelayedJob-6.js +52 -23
  20. package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
  21. package/dist/cjs/scripts/addJobScheduler-11.js +5 -5
  22. package/dist/cjs/scripts/addParentJob-5.js +52 -23
  23. package/dist/cjs/scripts/addParentJob-5.js.map +1 -1
  24. package/dist/cjs/scripts/addPrioritizedJob-9.js +52 -23
  25. package/dist/cjs/scripts/addPrioritizedJob-9.js.map +1 -1
  26. package/dist/cjs/scripts/addRepeatableJob-2.js +5 -5
  27. package/dist/cjs/scripts/addStandardJob-9.js +52 -23
  28. package/dist/cjs/scripts/addStandardJob-9.js.map +1 -1
  29. package/dist/cjs/scripts/changePriority-7.js +5 -5
  30. package/dist/cjs/scripts/cleanJobsInSet-3.js +5 -5
  31. package/dist/cjs/scripts/drain-5.js +5 -5
  32. package/dist/cjs/scripts/{getRateLimitTtl-1.js → getRateLimitTtl-2.js} +7 -2
  33. package/dist/cjs/scripts/getRateLimitTtl-2.js.map +1 -0
  34. package/dist/cjs/scripts/index.js +1 -1
  35. package/dist/cjs/scripts/moveJobFromActiveToWait-9.js +6 -6
  36. package/dist/cjs/scripts/moveJobsToWait-8.js +5 -5
  37. package/dist/cjs/scripts/moveStalledJobsToWait-8.js +9 -9
  38. package/dist/cjs/scripts/moveToActive-11.js +13 -12
  39. package/dist/cjs/scripts/moveToActive-11.js.map +1 -1
  40. package/dist/cjs/scripts/moveToDelayed-8.js +1 -1
  41. package/dist/cjs/scripts/moveToFinished-14.js +16 -15
  42. package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
  43. package/dist/cjs/scripts/obliterate-2.js +5 -5
  44. package/dist/cjs/scripts/promote-9.js +5 -5
  45. package/dist/cjs/scripts/removeChildDependency-1.js +5 -5
  46. package/dist/cjs/scripts/removeJob-2.js +5 -5
  47. package/dist/cjs/scripts/removeUnprocessedChildren-2.js +5 -5
  48. package/dist/cjs/scripts/reprocessJob-8.js +5 -5
  49. package/dist/cjs/scripts/retryJob-11.js +5 -5
  50. package/dist/cjs/scripts/updateJobScheduler-12.js +5 -5
  51. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  52. package/dist/cjs/version.js +1 -1
  53. package/dist/esm/classes/child-pool.js +8 -1
  54. package/dist/esm/classes/child-pool.js.map +1 -1
  55. package/dist/esm/classes/queue.d.ts +6 -0
  56. package/dist/esm/classes/queue.js +9 -0
  57. package/dist/esm/classes/queue.js.map +1 -1
  58. package/dist/esm/classes/scripts.js +4 -1
  59. package/dist/esm/classes/scripts.js.map +1 -1
  60. package/dist/esm/classes/worker.d.ts +1 -0
  61. package/dist/esm/classes/worker.js +42 -30
  62. package/dist/esm/classes/worker.js.map +1 -1
  63. package/dist/esm/commands/changePriority-7.lua +1 -1
  64. package/dist/esm/commands/{getRateLimitTtl-1.lua → getRateLimitTtl-2.lua} +6 -0
  65. package/dist/esm/commands/includes/deduplicateJob.lua +53 -20
  66. package/dist/esm/commands/includes/getTargetQueueList.lua +5 -5
  67. package/dist/esm/commands/includes/prepareJobForProcessing.lua +2 -4
  68. package/dist/esm/commands/moveJobFromActiveToWait-9.lua +1 -1
  69. package/dist/esm/commands/moveStalledJobsToWait-8.lua +0 -1
  70. package/dist/esm/commands/moveToActive-11.lua +7 -4
  71. package/dist/esm/commands/moveToDelayed-8.lua +2 -1
  72. package/dist/esm/commands/moveToFinished-14.lua +10 -7
  73. package/dist/esm/interfaces/base-job-options.d.ts +1 -1
  74. package/dist/esm/interfaces/worker-options.d.ts +8 -0
  75. package/dist/esm/scripts/addDelayedJob-6.js +52 -23
  76. package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
  77. package/dist/esm/scripts/addJobScheduler-11.js +5 -5
  78. package/dist/esm/scripts/addParentJob-5.js +52 -23
  79. package/dist/esm/scripts/addParentJob-5.js.map +1 -1
  80. package/dist/esm/scripts/addPrioritizedJob-9.js +52 -23
  81. package/dist/esm/scripts/addPrioritizedJob-9.js.map +1 -1
  82. package/dist/esm/scripts/addRepeatableJob-2.js +5 -5
  83. package/dist/esm/scripts/addStandardJob-9.js +52 -23
  84. package/dist/esm/scripts/addStandardJob-9.js.map +1 -1
  85. package/dist/esm/scripts/changePriority-7.js +5 -5
  86. package/dist/esm/scripts/cleanJobsInSet-3.js +5 -5
  87. package/dist/esm/scripts/drain-5.js +5 -5
  88. package/dist/esm/scripts/{getRateLimitTtl-1.js → getRateLimitTtl-2.js} +7 -2
  89. package/dist/esm/scripts/getRateLimitTtl-2.js.map +1 -0
  90. package/dist/esm/scripts/index.d.ts +1 -1
  91. package/dist/esm/scripts/index.js +1 -1
  92. package/dist/esm/scripts/moveJobFromActiveToWait-9.js +6 -6
  93. package/dist/esm/scripts/moveJobsToWait-8.js +5 -5
  94. package/dist/esm/scripts/moveStalledJobsToWait-8.js +9 -9
  95. package/dist/esm/scripts/moveToActive-11.js +13 -12
  96. package/dist/esm/scripts/moveToActive-11.js.map +1 -1
  97. package/dist/esm/scripts/moveToDelayed-8.js +1 -1
  98. package/dist/esm/scripts/moveToFinished-14.js +16 -15
  99. package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
  100. package/dist/esm/scripts/obliterate-2.js +5 -5
  101. package/dist/esm/scripts/promote-9.js +5 -5
  102. package/dist/esm/scripts/removeChildDependency-1.js +5 -5
  103. package/dist/esm/scripts/removeJob-2.js +5 -5
  104. package/dist/esm/scripts/removeUnprocessedChildren-2.js +5 -5
  105. package/dist/esm/scripts/reprocessJob-8.js +5 -5
  106. package/dist/esm/scripts/retryJob-11.js +5 -5
  107. package/dist/esm/scripts/updateJobScheduler-12.js +5 -5
  108. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  109. package/dist/esm/version.d.ts +1 -1
  110. package/dist/esm/version.js +1 -1
  111. package/package.json +1 -1
  112. package/dist/cjs/scripts/getRateLimitTtl-1.js.map +0 -1
  113. package/dist/esm/scripts/getRateLimitTtl-1.js.map +0 -1
  114. /package/dist/esm/scripts/{getRateLimitTtl-1.d.ts → getRateLimitTtl-2.d.ts} +0 -0
@@ -50,7 +50,7 @@ if rcall("EXISTS", jobKey) == 1 then
50
50
  local prioritizedKey = KEYS[4]
51
51
  local priorityCounterKey = KEYS[6]
52
52
  local markerKey = KEYS[7]
53
-
53
+
54
54
  -- Re-add with the new priority
55
55
  if rcall("ZREM", prioritizedKey, jobId) > 0 then
56
56
  reAddJobWithNewPriority( prioritizedKey, markerKey, target,
@@ -3,6 +3,7 @@
3
3
 
4
4
  Input:
5
5
  KEYS[1] 'limiter'
6
+ KEYS[2] 'meta'
6
7
 
7
8
  ARGV[1] maxJobs
8
9
  ]]
@@ -16,5 +17,10 @@ local rateLimiterKey = KEYS[1]
16
17
  if ARGV[1] ~= "0" then
17
18
  return getRateLimitTTL(tonumber(ARGV[1]), rateLimiterKey)
18
19
  else
20
+ local rateLimitMax = rcall("HGET", KEYS[2], "max")
21
+ if rateLimitMax then
22
+ return getRateLimitTTL(tonumber(rateLimitMax), rateLimiterKey)
23
+ end
24
+
19
25
  return rcall("PTTL", rateLimiterKey)
20
26
  end
@@ -3,39 +3,71 @@
3
3
  ]]
4
4
  -- Includes
5
5
  --- @include "removeJobKeys"
6
+
7
+ local function removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents, currentDeduplicatedJobId,
8
+ jobId, deduplicationId, prefix)
9
+ if rcall("ZREM", delayedKey, currentDeduplicatedJobId) > 0 then
10
+ removeJobKeys(prefix .. currentDeduplicatedJobId)
11
+ rcall("XADD", eventsKey, "*", "event", "removed", "jobId", currentDeduplicatedJobId,
12
+ "prev", "delayed")
13
+
14
+ -- TODO remove debounced event in next breaking change
15
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId",
16
+ jobId, "debounceId", deduplicationId)
17
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
18
+ jobId, "deduplicationId", deduplicationId, "deduplicatedJobId", currentDeduplicatedJobId)
19
+
20
+ return true
21
+ end
22
+
23
+ return false
24
+ end
25
+
6
26
  local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicationKey, eventsKey, maxEvents,
7
27
  prefix)
8
28
  local deduplicationId = deduplicationOpts and deduplicationOpts['id']
9
29
  if deduplicationId then
10
30
  local ttl = deduplicationOpts['ttl']
11
- if deduplicationOpts['replace'] and ttl and ttl > 0 then
12
- local currentDebounceJobId = rcall('GET', deduplicationKey)
13
- if currentDebounceJobId then
14
- if rcall("ZREM", delayedKey, currentDebounceJobId) > 0 then
15
- removeJobKeys(prefix .. currentDebounceJobId)
16
- rcall("XADD", eventsKey, "*", "event", "removed", "jobId", currentDebounceJobId,
17
- "prev", "delayed")
18
-
19
- if deduplicationOpts['extend'] then
20
- rcall('SET', deduplicationKey, jobId, 'PX', ttl)
31
+ if deduplicationOpts['replace'] then
32
+ if ttl and ttl > 0 then
33
+ local currentDebounceJobId = rcall('GET', deduplicationKey)
34
+ if currentDebounceJobId then
35
+ local isRemoved = removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents,
36
+ currentDebounceJobId, jobId, deduplicationId, prefix)
37
+ if isRemoved then
38
+ if deduplicationOpts['extend'] then
39
+ rcall('SET', deduplicationKey, jobId, 'PX', ttl)
40
+ else
41
+ rcall('SET', deduplicationKey, jobId, 'KEEPTTL')
42
+ end
43
+ return
21
44
  else
22
- rcall('SET', deduplicationKey, jobId, 'KEEPTTL')
45
+ return currentDebounceJobId
23
46
  end
24
-
25
- rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
26
- jobId, "deduplicationId", deduplicationId, "deduplicatedJobId", currentDebounceJobId)
27
- return
28
47
  else
29
- return currentDebounceJobId
48
+ rcall('SET', deduplicationKey, jobId, 'PX', ttl)
49
+ return
30
50
  end
31
51
  else
32
- rcall('SET', deduplicationKey, jobId, 'PX', ttl)
33
- return
52
+ local currentDebounceJobId = rcall('GET', deduplicationKey)
53
+ if currentDebounceJobId then
54
+ local isRemoved = removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents,
55
+ currentDebounceJobId, jobId, deduplicationId, prefix)
56
+
57
+ if isRemoved then
58
+ rcall('SET', deduplicationKey, jobId)
59
+ return
60
+ else
61
+ return currentDebounceJobId
62
+ end
63
+ else
64
+ rcall('SET', deduplicationKey, jobId)
65
+ return
66
+ end
34
67
  end
35
68
  else
36
- local ttl = deduplicationOpts['ttl']
37
69
  local deduplicationKeyExists
38
- if ttl then
70
+ if ttl and ttl > 0 then
39
71
  if deduplicationOpts['extend'] then
40
72
  local currentDebounceJobId = rcall('GET', deduplicationKey)
41
73
  if currentDebounceJobId then
@@ -58,6 +90,7 @@ local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicatio
58
90
 
59
91
  if deduplicationKeyExists then
60
92
  local currentDebounceJobId = rcall('GET', deduplicationKey)
93
+ -- TODO remove debounced event in next breaking change
61
94
  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId",
62
95
  currentDebounceJobId, "debounceId", deduplicationId)
63
96
  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
@@ -4,19 +4,19 @@
4
4
  ]]
5
5
 
6
6
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
7
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
7
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
8
8
 
9
9
  if queueAttributes[1] then
10
- return pausedKey, true
10
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
11
11
  else
12
12
  if queueAttributes[2] then
13
13
  local activeCount = rcall("LLEN", activeKey)
14
14
  if activeCount >= tonumber(queueAttributes[2]) then
15
- return waitKey, true
15
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
16
16
  else
17
- return waitKey, false
17
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
18
18
  end
19
19
  end
20
20
  end
21
- return waitKey, false
21
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
22
22
  end
@@ -10,7 +10,7 @@
10
10
  --- @include "addBaseMarkerIfNeeded"
11
11
 
12
12
  local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey,
13
- jobId, processedOn, maxJobs, markerKey, opts)
13
+ jobId, processedOn, maxJobs, limiterDuration, markerKey, opts)
14
14
  local jobKey = keyPrefix .. jobId
15
15
 
16
16
  -- Check if we need to perform rate limiting.
@@ -18,16 +18,14 @@ local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey
18
18
  local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
19
19
 
20
20
  if jobCounter == 1 then
21
- local limiterDuration = opts['limiter'] and opts['limiter']['duration']
22
21
  local integerDuration = math.floor(math.abs(limiterDuration))
23
22
  rcall("PEXPIRE", rateLimiterKey, integerDuration)
24
23
  end
25
24
  end
26
25
 
27
- local lockKey = jobKey .. ':lock'
28
-
29
26
  -- get a lock
30
27
  if opts['token'] ~= "0" then
28
+ local lockKey = jobKey .. ':lock'
31
29
  rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
32
30
  end
33
31
 
@@ -10,7 +10,7 @@
10
10
  KEYS[6] limiter key
11
11
  KEYS[7] prioritized key
12
12
  KEYS[8] marker key
13
- KEYS[9] event key
13
+ KEYS[9] event key
14
14
 
15
15
  ARGV[1] job id
16
16
  ARGV[2] lock token
@@ -24,7 +24,6 @@ local rcall = redis.call
24
24
  -- Includes
25
25
  --- @include "includes/addJobInTargetList"
26
26
  --- @include "includes/batches"
27
- --- @include "includes/getTargetQueueList"
28
27
  --- @include "includes/moveJobToWait"
29
28
  --- @include "includes/trimEvents"
30
29
 
@@ -51,14 +51,15 @@ local opts = cmsgpack.unpack(ARGV[3])
51
51
  --- @include "includes/prepareJobForProcessing"
52
52
  --- @include "includes/promoteDelayedJobs"
53
53
 
54
- local target, isPausedOrMaxed = getTargetQueueList(KEYS[9], activeKey, waitKey, KEYS[8])
54
+ local target, isPausedOrMaxed, rateLimitMax, rateLimitDuration = getTargetQueueList(KEYS[9],
55
+ activeKey, waitKey, KEYS[8])
55
56
 
56
57
  -- Check if there are delayed jobs that we can move to wait.
57
58
  local markerKey = KEYS[11]
58
59
  promoteDelayedJobs(delayedKey, markerKey, target, KEYS[3], eventStreamKey, ARGV[1],
59
60
  ARGV[2], KEYS[10], isPausedOrMaxed)
60
61
 
61
- local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
62
+ local maxJobs = tonumber(rateLimitMax or (opts['limiter'] and opts['limiter']['max']))
62
63
  local expireTime = getRateLimitTTL(maxJobs, rateLimiterKey)
63
64
 
64
65
  -- Check if we are rate limited first.
@@ -67,6 +68,8 @@ if expireTime > 0 then return {0, 0, expireTime, 0} end
67
68
  -- paused or maxed queue
68
69
  if isPausedOrMaxed then return {0, 0, 0, 0} end
69
70
 
71
+ local limiterDuration = (opts['limiter'] and opts['limiter']['duration']) or rateLimitDuration
72
+
70
73
  -- no job ID, try non-blocking move from wait to active
71
74
  local jobId = rcall("RPOPLPUSH", waitKey, activeKey)
72
75
 
@@ -78,12 +81,12 @@ end
78
81
 
79
82
  if jobId then
80
83
  return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
81
- maxJobs, markerKey, opts)
84
+ maxJobs, limiterDuration, markerKey, opts)
82
85
  else
83
86
  jobId = moveJobFromPrioritizedToActive(KEYS[3], activeKey, KEYS[10])
84
87
  if jobId then
85
88
  return prepareJobForProcessing(ARGV[1], rateLimiterKey, eventStreamKey, jobId, ARGV[2],
86
- maxJobs, markerKey, opts)
89
+ maxJobs, limiterDuration, markerKey, opts)
87
90
  end
88
91
  end
89
92
 
@@ -50,11 +50,12 @@ if rcall("EXISTS", jobKey) == 1 then
50
50
  local delayedKey = KEYS[4]
51
51
  local jobId = ARGV[3]
52
52
  local delay = tonumber(ARGV[5])
53
- local score, delayedTimestamp = getDelayedScore(delayedKey, ARGV[2], delay)
54
53
 
55
54
  local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
56
55
  if numRemovedElements < 1 then return -3 end
57
56
 
57
+ local score, delayedTimestamp = getDelayedScore(delayedKey, ARGV[2], delay)
58
+
58
59
  if ARGV[6] == "0" then
59
60
  rcall("HINCRBY", jobKey, "atm", 1)
60
61
  end
@@ -209,14 +209,15 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
209
209
  -- and not rate limited.
210
210
  if (ARGV[6] == "1") then
211
211
 
212
- local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[2], KEYS[1], KEYS[8])
212
+ local target, isPausedOrMaxed, rateLimitMax, rateLimitDuration = getTargetQueueList(metaKey, KEYS[2],
213
+ KEYS[1], KEYS[8])
213
214
 
214
215
  local markerKey = KEYS[14]
215
216
  -- Check if there are delayed jobs that can be promoted
216
217
  promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[3], eventStreamKey, prefix, timestamp, KEYS[10],
217
218
  isPausedOrMaxed)
218
219
 
219
- local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
220
+ local maxJobs = tonumber(rateLimitMax or (opts['limiter'] and opts['limiter']['max']))
220
221
  -- Check if we are rate limited first.
221
222
  local expireTime = getRateLimitTTL(maxJobs, KEYS[6])
222
223
 
@@ -229,6 +230,8 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
229
230
  return {0, 0, 0, 0}
230
231
  end
231
232
 
233
+ local limiterDuration = (opts['limiter'] and opts['limiter']['duration']) or rateLimitDuration
234
+
232
235
  jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
233
236
 
234
237
  if jobId then
@@ -241,17 +244,17 @@ if rcall("EXISTS", jobIdKey) == 1 then -- Make sure job exists
241
244
  if jobId == "0:0" then
242
245
  jobId = moveJobFromPrioritizedToActive(KEYS[3], KEYS[2], KEYS[10])
243
246
  return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
244
- markerKey, opts)
247
+ limiterDuration, markerKey, opts)
245
248
  end
246
249
  else
247
- return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs, markerKey,
248
- opts)
250
+ return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
251
+ limiterDuration, markerKey, opts)
249
252
  end
250
253
  else
251
254
  jobId = moveJobFromPrioritizedToActive(KEYS[3], KEYS[2], KEYS[10])
252
255
  if jobId then
253
- return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs, markerKey,
254
- opts)
256
+ return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId, timestamp, maxJobs,
257
+ limiterDuration, markerKey, opts)
255
258
  end
256
259
  end
257
260
 
@@ -123,37 +123,65 @@ local function removeJobKeys(jobKey)
123
123
  return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
124
124
  jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
125
125
  end
126
+ local function removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents, currentDeduplicatedJobId,
127
+ jobId, deduplicationId, prefix)
128
+ if rcall("ZREM", delayedKey, currentDeduplicatedJobId) > 0 then
129
+ removeJobKeys(prefix .. currentDeduplicatedJobId)
130
+ rcall("XADD", eventsKey, "*", "event", "removed", "jobId", currentDeduplicatedJobId,
131
+ "prev", "delayed")
132
+ -- TODO remove debounced event in next breaking change
133
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId",
134
+ jobId, "debounceId", deduplicationId)
135
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
136
+ jobId, "deduplicationId", deduplicationId, "deduplicatedJobId", currentDeduplicatedJobId)
137
+ return true
138
+ end
139
+ return false
140
+ end
126
141
  local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicationKey, eventsKey, maxEvents,
127
142
  prefix)
128
143
  local deduplicationId = deduplicationOpts and deduplicationOpts['id']
129
144
  if deduplicationId then
130
145
  local ttl = deduplicationOpts['ttl']
131
- if deduplicationOpts['replace'] and ttl and ttl > 0 then
132
- local currentDebounceJobId = rcall('GET', deduplicationKey)
133
- if currentDebounceJobId then
134
- if rcall("ZREM", delayedKey, currentDebounceJobId) > 0 then
135
- removeJobKeys(prefix .. currentDebounceJobId)
136
- rcall("XADD", eventsKey, "*", "event", "removed", "jobId", currentDebounceJobId,
137
- "prev", "delayed")
138
- if deduplicationOpts['extend'] then
139
- rcall('SET', deduplicationKey, jobId, 'PX', ttl)
146
+ if deduplicationOpts['replace'] then
147
+ if ttl and ttl > 0 then
148
+ local currentDebounceJobId = rcall('GET', deduplicationKey)
149
+ if currentDebounceJobId then
150
+ local isRemoved = removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents,
151
+ currentDebounceJobId, jobId, deduplicationId, prefix)
152
+ if isRemoved then
153
+ if deduplicationOpts['extend'] then
154
+ rcall('SET', deduplicationKey, jobId, 'PX', ttl)
155
+ else
156
+ rcall('SET', deduplicationKey, jobId, 'KEEPTTL')
157
+ end
158
+ return
140
159
  else
141
- rcall('SET', deduplicationKey, jobId, 'KEEPTTL')
160
+ return currentDebounceJobId
142
161
  end
143
- rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
144
- jobId, "deduplicationId", deduplicationId, "deduplicatedJobId", currentDebounceJobId)
145
- return
146
162
  else
147
- return currentDebounceJobId
163
+ rcall('SET', deduplicationKey, jobId, 'PX', ttl)
164
+ return
148
165
  end
149
166
  else
150
- rcall('SET', deduplicationKey, jobId, 'PX', ttl)
151
- return
167
+ local currentDebounceJobId = rcall('GET', deduplicationKey)
168
+ if currentDebounceJobId then
169
+ local isRemoved = removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents,
170
+ currentDebounceJobId, jobId, deduplicationId, prefix)
171
+ if isRemoved then
172
+ rcall('SET', deduplicationKey, jobId)
173
+ return
174
+ else
175
+ return currentDebounceJobId
176
+ end
177
+ else
178
+ rcall('SET', deduplicationKey, jobId)
179
+ return
180
+ end
152
181
  end
153
182
  else
154
- local ttl = deduplicationOpts['ttl']
155
183
  local deduplicationKeyExists
156
- if ttl then
184
+ if ttl and ttl > 0 then
157
185
  if deduplicationOpts['extend'] then
158
186
  local currentDebounceJobId = rcall('GET', deduplicationKey)
159
187
  if currentDebounceJobId then
@@ -175,6 +203,7 @@ local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicatio
175
203
  end
176
204
  if deduplicationKeyExists then
177
205
  local currentDebounceJobId = rcall('GET', deduplicationKey)
206
+ -- TODO remove debounced event in next breaking change
178
207
  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId",
179
208
  currentDebounceJobId, "debounceId", deduplicationId)
180
209
  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
@@ -277,20 +306,20 @@ end
277
306
  (since an empty list and !EXISTS are not really the same).
278
307
  ]]
279
308
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
280
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
309
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
281
310
  if queueAttributes[1] then
282
- return pausedKey, true
311
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
283
312
  else
284
313
  if queueAttributes[2] then
285
314
  local activeCount = rcall("LLEN", activeKey)
286
315
  if activeCount >= tonumber(queueAttributes[2]) then
287
- return waitKey, true
316
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
288
317
  else
289
- return waitKey, false
318
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
290
319
  end
291
320
  end
292
321
  end
293
- return waitKey, false
322
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
294
323
  end
295
324
  local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
296
325
  local parentWaitKey = parentQueueKey .. ":wait"
@@ -1 +1 @@
1
- {"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0bf,CAAC;AACW,QAAA,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAudf,CAAC;AACW,QAAA,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -182,20 +182,20 @@ end
182
182
  (since an empty list and !EXISTS are not really the same).
183
183
  ]]
184
184
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
185
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
185
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
186
186
  if queueAttributes[1] then
187
- return pausedKey, true
187
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
188
188
  else
189
189
  if queueAttributes[2] then
190
190
  local activeCount = rcall("LLEN", activeKey)
191
191
  if activeCount >= tonumber(queueAttributes[2]) then
192
- return waitKey, true
192
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
193
193
  else
194
- return waitKey, false
194
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
195
195
  end
196
196
  end
197
197
  end
198
- return waitKey, false
198
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
199
199
  end
200
200
  --[[
201
201
  Function to add job in target list and add marker if needed.
@@ -56,37 +56,65 @@ local function removeJobKeys(jobKey)
56
56
  return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
57
57
  jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
58
58
  end
59
+ local function removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents, currentDeduplicatedJobId,
60
+ jobId, deduplicationId, prefix)
61
+ if rcall("ZREM", delayedKey, currentDeduplicatedJobId) > 0 then
62
+ removeJobKeys(prefix .. currentDeduplicatedJobId)
63
+ rcall("XADD", eventsKey, "*", "event", "removed", "jobId", currentDeduplicatedJobId,
64
+ "prev", "delayed")
65
+ -- TODO remove debounced event in next breaking change
66
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId",
67
+ jobId, "debounceId", deduplicationId)
68
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
69
+ jobId, "deduplicationId", deduplicationId, "deduplicatedJobId", currentDeduplicatedJobId)
70
+ return true
71
+ end
72
+ return false
73
+ end
59
74
  local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicationKey, eventsKey, maxEvents,
60
75
  prefix)
61
76
  local deduplicationId = deduplicationOpts and deduplicationOpts['id']
62
77
  if deduplicationId then
63
78
  local ttl = deduplicationOpts['ttl']
64
- if deduplicationOpts['replace'] and ttl and ttl > 0 then
65
- local currentDebounceJobId = rcall('GET', deduplicationKey)
66
- if currentDebounceJobId then
67
- if rcall("ZREM", delayedKey, currentDebounceJobId) > 0 then
68
- removeJobKeys(prefix .. currentDebounceJobId)
69
- rcall("XADD", eventsKey, "*", "event", "removed", "jobId", currentDebounceJobId,
70
- "prev", "delayed")
71
- if deduplicationOpts['extend'] then
72
- rcall('SET', deduplicationKey, jobId, 'PX', ttl)
79
+ if deduplicationOpts['replace'] then
80
+ if ttl and ttl > 0 then
81
+ local currentDebounceJobId = rcall('GET', deduplicationKey)
82
+ if currentDebounceJobId then
83
+ local isRemoved = removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents,
84
+ currentDebounceJobId, jobId, deduplicationId, prefix)
85
+ if isRemoved then
86
+ if deduplicationOpts['extend'] then
87
+ rcall('SET', deduplicationKey, jobId, 'PX', ttl)
88
+ else
89
+ rcall('SET', deduplicationKey, jobId, 'KEEPTTL')
90
+ end
91
+ return
73
92
  else
74
- rcall('SET', deduplicationKey, jobId, 'KEEPTTL')
93
+ return currentDebounceJobId
75
94
  end
76
- rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
77
- jobId, "deduplicationId", deduplicationId, "deduplicatedJobId", currentDebounceJobId)
78
- return
79
95
  else
80
- return currentDebounceJobId
96
+ rcall('SET', deduplicationKey, jobId, 'PX', ttl)
97
+ return
81
98
  end
82
99
  else
83
- rcall('SET', deduplicationKey, jobId, 'PX', ttl)
84
- return
100
+ local currentDebounceJobId = rcall('GET', deduplicationKey)
101
+ if currentDebounceJobId then
102
+ local isRemoved = removeDelayedJob(delayedKey, deduplicationKey, eventsKey, maxEvents,
103
+ currentDebounceJobId, jobId, deduplicationId, prefix)
104
+ if isRemoved then
105
+ rcall('SET', deduplicationKey, jobId)
106
+ return
107
+ else
108
+ return currentDebounceJobId
109
+ end
110
+ else
111
+ rcall('SET', deduplicationKey, jobId)
112
+ return
113
+ end
85
114
  end
86
115
  else
87
- local ttl = deduplicationOpts['ttl']
88
116
  local deduplicationKeyExists
89
- if ttl then
117
+ if ttl and ttl > 0 then
90
118
  if deduplicationOpts['extend'] then
91
119
  local currentDebounceJobId = rcall('GET', deduplicationKey)
92
120
  if currentDebounceJobId then
@@ -108,6 +136,7 @@ local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicatio
108
136
  end
109
137
  if deduplicationKeyExists then
110
138
  local currentDebounceJobId = rcall('GET', deduplicationKey)
139
+ -- TODO remove debounced event in next breaking change
111
140
  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId",
112
141
  currentDebounceJobId, "debounceId", deduplicationId)
113
142
  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
@@ -234,20 +263,20 @@ end
234
263
  (since an empty list and !EXISTS are not really the same).
235
264
  ]]
236
265
  local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
237
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
266
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency", "max", "duration")
238
267
  if queueAttributes[1] then
239
- return pausedKey, true
268
+ return pausedKey, true, queueAttributes[3], queueAttributes[4]
240
269
  else
241
270
  if queueAttributes[2] then
242
271
  local activeCount = rcall("LLEN", activeKey)
243
272
  if activeCount >= tonumber(queueAttributes[2]) then
244
- return waitKey, true
273
+ return waitKey, true, queueAttributes[3], queueAttributes[4]
245
274
  else
246
- return waitKey, false
275
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
247
276
  end
248
277
  end
249
278
  end
250
- return waitKey, false
279
+ return waitKey, false, queueAttributes[3], queueAttributes[4]
251
280
  end
252
281
  local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
253
282
  local parentWaitKey = parentQueueKey .. ":wait"
@@ -1 +1 @@
1
- {"version":3,"file":"addParentJob-5.js","sourceRoot":"","sources":["../../../src/scripts/addParentJob-5.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkZf,CAAC;AACW,QAAA,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"addParentJob-5.js","sourceRoot":"","sources":["../../../src/scripts/addParentJob-5.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+af,CAAC;AACW,QAAA,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}