bullmq 5.10.3 → 5.11.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (132) hide show
  1. package/dist/cjs/classes/job.js +6 -0
  2. package/dist/cjs/classes/job.js.map +1 -1
  3. package/dist/cjs/classes/queue-events.js.map +1 -1
  4. package/dist/cjs/classes/queue-keys.js +2 -1
  5. package/dist/cjs/classes/queue-keys.js.map +1 -1
  6. package/dist/cjs/classes/queue.js +9 -0
  7. package/dist/cjs/classes/queue.js.map +1 -1
  8. package/dist/cjs/classes/scripts.js +2 -1
  9. package/dist/cjs/classes/scripts.js.map +1 -1
  10. package/dist/cjs/commands/addDelayedJob-6.lua +11 -1
  11. package/dist/cjs/commands/addParentJob-4.lua +10 -1
  12. package/dist/cjs/commands/addPrioritizedJob-8.lua +11 -2
  13. package/dist/cjs/commands/addStandardJob-8.lua +11 -2
  14. package/dist/cjs/commands/includes/cleanList.lua +1 -1
  15. package/dist/cjs/commands/includes/cleanSet.lua +2 -2
  16. package/dist/cjs/commands/includes/debounceJob.lua +23 -0
  17. package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +6 -3
  18. package/dist/cjs/commands/includes/removeDebounceKey.lua +12 -0
  19. package/dist/cjs/commands/includes/removeDebounceKeyIfNeeded.lua +14 -0
  20. package/dist/cjs/commands/includes/removeJob.lua +5 -1
  21. package/dist/cjs/commands/includes/removeJobs.lua +1 -1
  22. package/dist/cjs/commands/includes/removeJobsByMaxAge.lua +3 -2
  23. package/dist/cjs/commands/includes/removeJobsByMaxCount.lua +1 -1
  24. package/dist/cjs/commands/includes/removeParentDependencyKey.lua +11 -4
  25. package/dist/cjs/commands/includes/storeJob.lua +7 -1
  26. package/dist/cjs/commands/moveStalledJobsToWait-9.lua +6 -2
  27. package/dist/cjs/commands/moveToFinished-14.lua +14 -11
  28. package/dist/cjs/commands/removeChildDependency-1.lua +1 -1
  29. package/dist/cjs/commands/removeJob-2.lua +3 -1
  30. package/dist/cjs/commands/{removeRepeatable-2.lua → removeRepeatable-3.lua} +4 -2
  31. package/dist/cjs/interfaces/debounce-options.js +3 -0
  32. package/dist/cjs/interfaces/debounce-options.js.map +1 -0
  33. package/dist/cjs/interfaces/index.js +1 -0
  34. package/dist/cjs/interfaces/index.js.map +1 -1
  35. package/dist/cjs/scripts/addDelayedJob-6.js +34 -1
  36. package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
  37. package/dist/cjs/scripts/addParentJob-4.js +34 -1
  38. package/dist/cjs/scripts/addParentJob-4.js.map +1 -1
  39. package/dist/cjs/scripts/addPrioritizedJob-8.js +34 -1
  40. package/dist/cjs/scripts/addPrioritizedJob-8.js.map +1 -1
  41. package/dist/cjs/scripts/addStandardJob-8.js +34 -1
  42. package/dist/cjs/scripts/addStandardJob-8.js.map +1 -1
  43. package/dist/cjs/scripts/cleanJobsInSet-2.js +28 -8
  44. package/dist/cjs/scripts/cleanJobsInSet-2.js.map +1 -1
  45. package/dist/cjs/scripts/drain-4.js +26 -6
  46. package/dist/cjs/scripts/drain-4.js.map +1 -1
  47. package/dist/cjs/scripts/index.js +1 -1
  48. package/dist/cjs/scripts/moveStalledJobsToWait-9.js +49 -13
  49. package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
  50. package/dist/cjs/scripts/moveToFinished-14.js +56 -21
  51. package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
  52. package/dist/cjs/scripts/obliterate-2.js +26 -6
  53. package/dist/cjs/scripts/obliterate-2.js.map +1 -1
  54. package/dist/cjs/scripts/removeChildDependency-1.js +12 -5
  55. package/dist/cjs/scripts/removeChildDependency-1.js.map +1 -1
  56. package/dist/cjs/scripts/removeJob-2.js +23 -5
  57. package/dist/cjs/scripts/removeJob-2.js.map +1 -1
  58. package/dist/cjs/scripts/{removeRepeatable-2.js → removeRepeatable-3.js} +6 -4
  59. package/dist/cjs/scripts/removeRepeatable-3.js.map +1 -0
  60. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  61. package/dist/esm/classes/job.d.ts +4 -0
  62. package/dist/esm/classes/job.js +6 -0
  63. package/dist/esm/classes/job.js.map +1 -1
  64. package/dist/esm/classes/queue-events.d.ts +8 -0
  65. package/dist/esm/classes/queue-events.js.map +1 -1
  66. package/dist/esm/classes/queue-keys.js +2 -1
  67. package/dist/esm/classes/queue-keys.js.map +1 -1
  68. package/dist/esm/classes/queue.d.ts +6 -0
  69. package/dist/esm/classes/queue.js +9 -0
  70. package/dist/esm/classes/queue.js.map +1 -1
  71. package/dist/esm/classes/scripts.js +2 -1
  72. package/dist/esm/classes/scripts.js.map +1 -1
  73. package/dist/esm/commands/addDelayedJob-6.lua +11 -1
  74. package/dist/esm/commands/addParentJob-4.lua +10 -1
  75. package/dist/esm/commands/addPrioritizedJob-8.lua +11 -2
  76. package/dist/esm/commands/addStandardJob-8.lua +11 -2
  77. package/dist/esm/commands/includes/cleanList.lua +1 -1
  78. package/dist/esm/commands/includes/cleanSet.lua +2 -2
  79. package/dist/esm/commands/includes/debounceJob.lua +23 -0
  80. package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +6 -3
  81. package/dist/esm/commands/includes/removeDebounceKey.lua +12 -0
  82. package/dist/esm/commands/includes/removeDebounceKeyIfNeeded.lua +14 -0
  83. package/dist/esm/commands/includes/removeJob.lua +5 -1
  84. package/dist/esm/commands/includes/removeJobs.lua +1 -1
  85. package/dist/esm/commands/includes/removeJobsByMaxAge.lua +3 -2
  86. package/dist/esm/commands/includes/removeJobsByMaxCount.lua +1 -1
  87. package/dist/esm/commands/includes/removeParentDependencyKey.lua +11 -4
  88. package/dist/esm/commands/includes/storeJob.lua +7 -1
  89. package/dist/esm/commands/moveStalledJobsToWait-9.lua +6 -2
  90. package/dist/esm/commands/moveToFinished-14.lua +14 -11
  91. package/dist/esm/commands/removeChildDependency-1.lua +1 -1
  92. package/dist/esm/commands/removeJob-2.lua +3 -1
  93. package/dist/esm/commands/{removeRepeatable-2.lua → removeRepeatable-3.lua} +4 -2
  94. package/dist/esm/interfaces/debounce-options.d.ts +13 -0
  95. package/dist/esm/interfaces/debounce-options.js +2 -0
  96. package/dist/esm/interfaces/debounce-options.js.map +1 -0
  97. package/dist/esm/interfaces/index.d.ts +1 -0
  98. package/dist/esm/interfaces/index.js +1 -0
  99. package/dist/esm/interfaces/index.js.map +1 -1
  100. package/dist/esm/interfaces/job-json.d.ts +2 -0
  101. package/dist/esm/scripts/addDelayedJob-6.js +34 -1
  102. package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
  103. package/dist/esm/scripts/addParentJob-4.js +34 -1
  104. package/dist/esm/scripts/addParentJob-4.js.map +1 -1
  105. package/dist/esm/scripts/addPrioritizedJob-8.js +34 -1
  106. package/dist/esm/scripts/addPrioritizedJob-8.js.map +1 -1
  107. package/dist/esm/scripts/addStandardJob-8.js +34 -1
  108. package/dist/esm/scripts/addStandardJob-8.js.map +1 -1
  109. package/dist/esm/scripts/cleanJobsInSet-2.js +28 -8
  110. package/dist/esm/scripts/cleanJobsInSet-2.js.map +1 -1
  111. package/dist/esm/scripts/drain-4.js +26 -6
  112. package/dist/esm/scripts/drain-4.js.map +1 -1
  113. package/dist/esm/scripts/index.d.ts +1 -1
  114. package/dist/esm/scripts/index.js +1 -1
  115. package/dist/esm/scripts/moveStalledJobsToWait-9.js +49 -13
  116. package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
  117. package/dist/esm/scripts/moveToFinished-14.js +56 -21
  118. package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
  119. package/dist/esm/scripts/obliterate-2.js +26 -6
  120. package/dist/esm/scripts/obliterate-2.js.map +1 -1
  121. package/dist/esm/scripts/removeChildDependency-1.js +12 -5
  122. package/dist/esm/scripts/removeChildDependency-1.js.map +1 -1
  123. package/dist/esm/scripts/removeJob-2.js +23 -5
  124. package/dist/esm/scripts/removeJob-2.js.map +1 -1
  125. package/dist/esm/scripts/{removeRepeatable-2.js → removeRepeatable-3.js} +6 -4
  126. package/dist/esm/scripts/removeRepeatable-3.js.map +1 -0
  127. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  128. package/dist/esm/types/job-options.d.ts +9 -1
  129. package/package.json +1 -1
  130. package/dist/cjs/scripts/removeRepeatable-2.js.map +0 -1
  131. package/dist/esm/scripts/removeRepeatable-2.js.map +0 -1
  132. /package/dist/esm/scripts/{removeRepeatable-2.d.ts → removeRepeatable-3.d.ts} +0 -0
@@ -0,0 +1,23 @@
1
+ --[[
2
+ Function to debounce a job.
3
+ ]]
4
+
5
+ local function debounceJob(prefixKey, debounceOpts, jobId, debounceKey, eventsKey, maxEvents)
6
+ local debounceId = debounceOpts and debounceOpts['id']
7
+ if debounceId then
8
+ local ttl = debounceOpts['ttl']
9
+ local debounceKeyExists
10
+ if ttl then
11
+ debounceKeyExists = not rcall('SET', debounceKey, jobId, 'PX', ttl, 'NX')
12
+ else
13
+ debounceKeyExists = not rcall('SET', debounceKey, jobId, 'NX')
14
+ end
15
+ if debounceKeyExists then
16
+ local currentDebounceJobId = rcall('GET', debounceKey)
17
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
18
+ "debounced", "jobId", currentDebounceJobId)
19
+ return currentDebounceJobId
20
+ end
21
+ end
22
+ end
23
+
@@ -4,6 +4,7 @@
4
4
 
5
5
  -- Includes
6
6
  --- @include "moveParentToWaitIfNeeded"
7
+ --- @include "removeDebounceKeyIfNeeded"
7
8
 
8
9
  local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
9
10
  if rcall("ZREM", parentQueueKey .. ":waiting-children", parentId) == 1 then
@@ -13,10 +14,12 @@ local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey,
13
14
  rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason",
14
15
  failedReason, "prev", "waiting-children")
15
16
 
16
- local rawParentData = rcall("HGET", parentKey, "parent")
17
+ local jobAttributes = rcall("HMGET", parentKey, "parent", "deid")
17
18
 
18
- if rawParentData ~= false then
19
- local parentData = cjson.decode(rawParentData)
19
+ removeDebounceKeyIfNeeded(parentQueueKey, jobAttributes[2])
20
+
21
+ if jobAttributes[1] then
22
+ local parentData = cjson.decode(jobAttributes[1])
20
23
  if parentData['fpof'] then
21
24
  moveParentFromWaitingChildrenToFailed(
22
25
  parentData['queueKey'],
@@ -0,0 +1,12 @@
1
+ --[[
2
+ Function to remove debounce key.
3
+ ]]
4
+
5
+ local function removeDebounceKey(prefixKey, jobKey)
6
+ local debounceId = rcall("HGET", jobKey, "deid")
7
+ if debounceId then
8
+ local debounceKey = prefixKey .. "de:" .. debounceId
9
+ rcall("DEL", debounceKey)
10
+ end
11
+ end
12
+
@@ -0,0 +1,14 @@
1
+ --[[
2
+ Function to remove debounce key if needed.
3
+ ]]
4
+
5
+ local function removeDebounceKeyIfNeeded(prefixKey, debounceId)
6
+ if debounceId then
7
+ local debounceKey = prefixKey .. "de:" .. debounceId
8
+ local pttl = rcall("PTTL", debounceKey)
9
+
10
+ if pttl == 0 or pttl == -1 then
11
+ rcall("DEL", debounceKey)
12
+ end
13
+ end
14
+ end
@@ -3,11 +3,15 @@
3
3
  ]]
4
4
 
5
5
  -- Includes
6
+ --- @include "removeDebounceKey"
6
7
  --- @include "removeJobKeys"
7
8
  --- @include "removeParentDependencyKey"
8
9
 
9
- local function removeJob(jobId, hard, baseKey)
10
+ local function removeJob(jobId, hard, baseKey, shouldRemoveDebounceKey)
10
11
  local jobKey = baseKey .. jobId
11
12
  removeParentDependencyKey(jobKey, hard, nil, baseKey)
13
+ if shouldRemoveDebounceKey then
14
+ removeDebounceKey(baseKey, jobKey)
15
+ end
12
16
  removeJobKeys(jobKey)
13
17
  end
@@ -7,7 +7,7 @@
7
7
 
8
8
  local function removeJobs(keys, hard, baseKey, max)
9
9
  for i, key in ipairs(keys) do
10
- removeJob(key, hard, baseKey)
10
+ removeJob(key, hard, baseKey, true --[[remove debounce key]])
11
11
  end
12
12
  return max - #keys
13
13
  end
@@ -5,11 +5,12 @@
5
5
  -- Includes
6
6
  --- @include "removeJob"
7
7
 
8
- local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
8
+ local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix,
9
+ shouldRemoveDebounceKey)
9
10
  local start = timestamp - maxAge * 1000
10
11
  local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
11
12
  for i, jobId in ipairs(jobIds) do
12
- removeJob(jobId, false, prefix)
13
+ removeJob(jobId, false, prefix, false --[[remove debounce key]])
13
14
  end
14
15
  rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
15
16
  end
@@ -9,7 +9,7 @@ local function removeJobsByMaxCount(maxCount, targetSet, prefix)
9
9
  local start = maxCount
10
10
  local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
11
11
  for i, jobId in ipairs(jobIds) do
12
- removeJob(jobId, false, prefix)
12
+ removeJob(jobId, false, prefix, false --[[remove debounce key]])
13
13
  end
14
14
  rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
15
15
  end
@@ -21,7 +21,7 @@ local function moveParentToWait(parentPrefix, parentId, emitEvent)
21
21
  end
22
22
  end
23
23
 
24
- local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
24
+ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debounceId)
25
25
  if parentKey then
26
26
  local parentDependenciesKey = parentKey .. ":dependencies"
27
27
  local result = rcall("SREM", parentDependenciesKey, jobKey)
@@ -36,8 +36,11 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
36
36
  if numRemovedElements == 1 then
37
37
  if hard then -- remove parent in same queue
38
38
  if parentPrefix == baseKey then
39
- removeParentDependencyKey(parentKey, hard, nil, baseKey)
39
+ removeParentDependencyKey(parentKey, hard, nil, baseKey, nil)
40
40
  removeJobKeys(parentKey)
41
+ if debounceId then
42
+ rcall("DEL", parentPrefix .. "de:" .. debounceId)
43
+ end
41
44
  else
42
45
  moveParentToWait(parentPrefix, parentId)
43
46
  end
@@ -49,7 +52,8 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
49
52
  return true
50
53
  end
51
54
  else
52
- local missedParentKey = rcall("HGET", jobKey, "parentKey")
55
+ local parentAttributes = rcall("HMGET", jobKey, "parentKey", "deid")
56
+ local missedParentKey = parentAttributes[1]
53
57
  if( (type(missedParentKey) == "string") and missedParentKey ~= ""
54
58
  and (rcall("EXISTS", missedParentKey) == 1)) then
55
59
  local parentDependenciesKey = missedParentKey .. ":dependencies"
@@ -65,8 +69,11 @@ local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
65
69
  if numRemovedElements == 1 then
66
70
  if hard then
67
71
  if parentPrefix == baseKey then
68
- removeParentDependencyKey(missedParentKey, hard, nil, baseKey)
72
+ removeParentDependencyKey(missedParentKey, hard, nil, baseKey, nil)
69
73
  removeJobKeys(missedParentKey)
74
+ if parentAttributes[2] then
75
+ rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
76
+ end
70
77
  else
71
78
  moveParentToWait(parentPrefix, parentId)
72
79
  end
@@ -6,7 +6,8 @@ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
6
6
  local jsonOpts = cjson.encode(opts)
7
7
  local delay = opts['delay'] or 0
8
8
  local priority = opts['priority'] or 0
9
-
9
+ local debounceId = opts['de'] and opts['de']['id']
10
+
10
11
  local optionalValues = {}
11
12
  if parentKey ~= nil then
12
13
  table.insert(optionalValues, "parentKey")
@@ -20,6 +21,11 @@ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
20
21
  table.insert(optionalValues, repeatJobKey)
21
22
  end
22
23
 
24
+ if debounceId then
25
+ table.insert(optionalValues, "deid")
26
+ table.insert(optionalValues, debounceId)
27
+ end
28
+
23
29
  rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
24
30
  "timestamp", timestamp, "delay", delay, "priority", priority,
25
31
  unpack(optionalValues))
@@ -29,6 +29,7 @@ local rcall = redis.call
29
29
  --- @include "includes/getTargetQueueList"
30
30
  --- @include "includes/moveParentFromWaitingChildrenToFailed"
31
31
  --- @include "includes/moveParentToWaitIfNeeded"
32
+ --- @include "includes/removeDebounceKeyIfNeeded"
32
33
  --- @include "includes/removeJob"
33
34
  --- @include "includes/removeJobsByMaxAge"
34
35
  --- @include "includes/removeJobsByMaxCount"
@@ -83,12 +84,14 @@ if (#stalling > 0) then
83
84
  local stalledCount =
84
85
  rcall("HINCRBY", jobKey, "stalledCounter", 1)
85
86
  if (stalledCount > MAX_STALLED_JOB_COUNT) then
86
- local jobAttributes = rcall("HMGET", jobKey, "opts", "parent")
87
+ local jobAttributes = rcall("HMGET", jobKey, "opts", "parent", "deid")
87
88
  local rawOpts = jobAttributes[1]
88
89
  local rawParentData = jobAttributes[2]
89
90
  local opts = cjson.decode(rawOpts)
90
91
  local removeOnFailType = type(opts["removeOnFail"])
91
92
  rcall("ZADD", failedKey, timestamp, jobId)
93
+ removeDebounceKeyIfNeeded(queueKeyPrefix, jobAttributes[3])
94
+
92
95
  local failedReason =
93
96
  "job stalled more than allowable limit"
94
97
  rcall("HMSET", jobKey, "failedReason", failedReason,
@@ -122,7 +125,8 @@ if (#stalling > 0) then
122
125
  failedKey, queueKeyPrefix)
123
126
  elseif removeOnFailType == "boolean" then
124
127
  if opts["removeOnFail"] then
125
- removeJob(jobId, false, queueKeyPrefix)
128
+ removeJob(jobId, false, queueKeyPrefix,
129
+ false --[[remove debounce key]])
126
130
  rcall("ZREM", failedKey, jobId)
127
131
  end
128
132
  elseif removeOnFailType ~= "nil" then
@@ -65,6 +65,7 @@ local rcall = redis.call
65
65
  --- @include "includes/moveParentToWaitIfNeeded"
66
66
  --- @include "includes/prepareJobForProcessing"
67
67
  --- @include "includes/promoteDelayedJobs"
68
+ --- @include "includes/removeDebounceKeyIfNeeded"
68
69
  --- @include "includes/removeJobKeys"
69
70
  --- @include "includes/removeJobsByMaxAge"
70
71
  --- @include "includes/removeJobsByMaxCount"
@@ -93,12 +94,12 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
93
94
  return -4
94
95
  end
95
96
 
96
- local parentReferences = rcall("HMGET", jobIdKey, "parentKey", "parent")
97
- local parentKey = parentReferences[1] or ""
97
+ local jobAttributes = rcall("HMGET", jobIdKey, "parentKey", "parent", "deid")
98
+ local parentKey = jobAttributes[1] or ""
98
99
  local parentId = ""
99
100
  local parentQueueKey = ""
100
- if parentReferences[2] ~= false then
101
- local jsonDecodedParent = cjson.decode(parentReferences[2])
101
+ if jobAttributes[2] ~= false then
102
+ local jsonDecodedParent = cjson.decode(jobAttributes[2])
102
103
  parentId = jsonDecodedParent['id']
103
104
  parentQueueKey = jsonDecodedParent['queueKey']
104
105
  end
@@ -116,6 +117,10 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
116
117
  -- Trim events before emiting them to avoid trimming events emitted in this script
117
118
  trimEvents(metaKey, eventStreamKey)
118
119
 
120
+ local prefix = ARGV[7]
121
+
122
+ removeDebounceKeyIfNeeded(prefix, jobAttributes[3])
123
+
119
124
  -- If job has a parent we need to
120
125
  -- 1) remove this job id from parents dependencies
121
126
  -- 2) move the job Id to parent "processed" set
@@ -164,8 +169,6 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
164
169
  -- "returnvalue" / "failedReason" and "finishedOn"
165
170
 
166
171
  -- Remove old jobs?
167
- local prefix = ARGV[7]
168
-
169
172
  if maxAge ~= nil then
170
173
  removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
171
174
  end
@@ -179,7 +182,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
179
182
  -- TODO: when a child is removed when finished, result or failure in parent
180
183
  -- must not be deleted, those value references should be deleted when the parent
181
184
  -- is deleted
182
- removeParentDependencyKey(jobIdKey, false, parentKey)
185
+ removeParentDependencyKey(jobIdKey, false, parentKey, jobAttributes[3])
183
186
  end
184
187
  end
185
188
 
@@ -205,7 +208,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
205
208
  local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[2], KEYS[1], KEYS[8])
206
209
 
207
210
  -- Check if there are delayed jobs that can be promoted
208
- promoteDelayedJobs(KEYS[7], KEYS[14], target, KEYS[3], eventStreamKey, ARGV[7],
211
+ promoteDelayedJobs(KEYS[7], KEYS[14], target, KEYS[3], eventStreamKey, prefix,
209
212
  timestamp, KEYS[10], isPausedOrMaxed)
210
213
 
211
214
  local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
@@ -229,19 +232,19 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
229
232
  if jobId == "0:0" then
230
233
  jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2],
231
234
  KEYS[10])
232
- return prepareJobForProcessing(ARGV[7], KEYS[6], eventStreamKey, jobId,
235
+ return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId,
233
236
  timestamp, maxJobs,
234
237
  opts)
235
238
  end
236
239
  else
237
- return prepareJobForProcessing(ARGV[7], KEYS[6], eventStreamKey, jobId,
240
+ return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId,
238
241
  timestamp, maxJobs,
239
242
  opts)
240
243
  end
241
244
  else
242
245
  jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
243
246
  if jobId then
244
- return prepareJobForProcessing(ARGV[7], KEYS[6], eventStreamKey, jobId,
247
+ return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId,
245
248
  timestamp, maxJobs,
246
249
  opts)
247
250
  end
@@ -25,7 +25,7 @@ if rcall("EXISTS", jobKey) ~= 1 then return -1 end
25
25
 
26
26
  if rcall("EXISTS", parentKey) ~= 1 then return -5 end
27
27
 
28
- if removeParentDependencyKey(jobKey, false, parentKey, KEYS[1]) then
28
+ if removeParentDependencyKey(jobKey, false, parentKey, KEYS[1], nil) then
29
29
  rcall("HDEL", jobKey, "parentKey", "parent")
30
30
 
31
31
  return 0
@@ -19,6 +19,7 @@ local rcall = redis.call
19
19
  --- @include "includes/destructureJobKey"
20
20
  --- @include "includes/getOrSetMaxEvents"
21
21
  --- @include "includes/isLocked"
22
+ --- @include "includes/removeDebounceKey"
22
23
  --- @include "includes/removeJobFromAnyState"
23
24
  --- @include "includes/removeJobKeys"
24
25
  --- @include "includes/removeParentDependencyKey"
@@ -26,7 +27,7 @@ local rcall = redis.call
26
27
  local function removeJob( prefix, jobId, parentKey, removeChildren)
27
28
  local jobKey = prefix .. jobId;
28
29
 
29
- removeParentDependencyKey(jobKey, false, parentKey)
30
+ removeParentDependencyKey(jobKey, false, parentKey, nil)
30
31
 
31
32
  if removeChildren == "1" then
32
33
  -- Check if this job has children
@@ -66,6 +67,7 @@ local function removeJob( prefix, jobId, parentKey, removeChildren)
66
67
 
67
68
  local prev = removeJobFromAnyState(prefix, jobId)
68
69
 
70
+ removeDebounceKey(prefix, jobKey)
69
71
  if removeJobKeys(jobKey) > 0 then
70
72
  local maxEvents = getOrSetMaxEvents(KEYS[2])
71
73
  rcall("XADD", prefix .. "events", "MAXLEN", "~", maxEvents, "*", "event", "removed",
@@ -4,6 +4,7 @@
4
4
  Input:
5
5
  KEYS[1] repeat jobs key
6
6
  KEYS[2] delayed jobs key
7
+ KEYS[3] events key
7
8
 
8
9
  ARGV[1] old repeat job id
9
10
  ARGV[2] options concat
@@ -29,7 +30,7 @@ if millis then
29
30
  local repeatJobId = ARGV[1] .. millis
30
31
  if(rcall("ZREM", KEYS[2], repeatJobId) == 1) then
31
32
  removeJobKeys(ARGV[4] .. repeatJobId)
32
- rcall("XADD", ARGV[4] .. "events", "*", "event", "removed", "jobId", repeatJobId, "prev", "delayed");
33
+ rcall("XADD", KEYS[3], "*", "event", "removed", "jobId", repeatJobId, "prev", "delayed");
33
34
  end
34
35
  end
35
36
 
@@ -45,11 +46,12 @@ if millis then
45
46
  local repeatJobId = "repeat:" .. ARGV[3] .. ":" .. millis
46
47
  if(rcall("ZREM", KEYS[2], repeatJobId) == 1) then
47
48
  removeJobKeys(ARGV[4] .. repeatJobId)
48
- rcall("XADD", ARGV[4] .. "events", "*", "event", "removed", "jobId", repeatJobId, "prev", "delayed");
49
+ rcall("XADD", KEYS[3], "*", "event", "removed", "jobId", repeatJobId, "prev", "delayed")
49
50
  end
50
51
  end
51
52
 
52
53
  if(rcall("ZREM", KEYS[1], ARGV[3]) == 1) then
54
+ rcall("DEL", KEYS[1] .. ":" .. ARGV[3])
53
55
  return 0
54
56
  end
55
57
 
@@ -0,0 +1,13 @@
1
+ /**
2
+ * Debounce options
3
+ */
4
+ export interface DebounceOptions {
5
+ /**
6
+ * ttl in milliseconds
7
+ */
8
+ ttl?: number;
9
+ /**
10
+ * Identifier
11
+ */
12
+ id: string;
13
+ }
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=debounce-options.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"debounce-options.js","sourceRoot":"","sources":["../../../src/interfaces/debounce-options.ts"],"names":[],"mappings":""}
@@ -3,6 +3,7 @@ export * from './backoff-options';
3
3
  export * from './base-job-options';
4
4
  export * from './child-message';
5
5
  export * from './connection';
6
+ export * from './debounce-options';
6
7
  export * from './flow-job';
7
8
  export * from './ioredis-events';
8
9
  export * from './job-json';
@@ -3,6 +3,7 @@ export * from './backoff-options';
3
3
  export * from './base-job-options';
4
4
  export * from './child-message';
5
5
  export * from './connection';
6
+ export * from './debounce-options';
6
7
  export * from './flow-job';
7
8
  export * from './ioredis-events';
8
9
  export * from './job-json';
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/interfaces/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,cAAc,CAAC;AAC7B,cAAc,YAAY,CAAC;AAC3B,cAAc,kBAAkB,CAAC;AACjC,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC;AAC5B,cAAc,mBAAmB,CAAC;AAClC,cAAc,WAAW,CAAC;AAC1B,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,UAAU,CAAC;AACzB,cAAc,iBAAiB,CAAC;AAChC,cAAc,wBAAwB,CAAC;AACvC,cAAc,iBAAiB,CAAC;AAChC,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC;AACjC,cAAc,sBAAsB,CAAC;AACrC,cAAc,kBAAkB,CAAC;AACjC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/interfaces/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC;AACnC,cAAc,mBAAmB,CAAC;AAClC,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,cAAc,CAAC;AAC7B,cAAc,oBAAoB,CAAC;AACnC,cAAc,YAAY,CAAC;AAC3B,cAAc,kBAAkB,CAAC;AACjC,cAAc,YAAY,CAAC;AAC3B,cAAc,aAAa,CAAC;AAC5B,cAAc,mBAAmB,CAAC;AAClC,cAAc,WAAW,CAAC;AAC1B,cAAc,eAAe,CAAC;AAC9B,cAAc,kBAAkB,CAAC;AACjC,cAAc,UAAU,CAAC;AACzB,cAAc,iBAAiB,CAAC;AAChC,cAAc,wBAAwB,CAAC;AACvC,cAAc,iBAAiB,CAAC;AAChC,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC;AACjC,cAAc,sBAAsB,CAAC;AACrC,cAAc,kBAAkB,CAAC;AACjC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,iBAAiB,CAAC;AAChC,cAAc,kBAAkB,CAAC"}
@@ -17,6 +17,7 @@ export interface JobJson {
17
17
  parent?: ParentKeys;
18
18
  parentKey?: string;
19
19
  repeatJobKey?: string;
20
+ debounceId?: string;
20
21
  processedBy?: string;
21
22
  }
22
23
  export interface JobJsonRaw {
@@ -35,6 +36,7 @@ export interface JobJsonRaw {
35
36
  returnvalue: string;
36
37
  parentKey?: string;
37
38
  parent?: string;
39
+ deid?: string;
38
40
  rjk?: string;
39
41
  atm?: string;
40
42
  ats?: string;
@@ -22,6 +22,7 @@ const content = `--[[
22
22
  [7] parent dependencies key.
23
23
  [8] parent? {id, queueKey}
24
24
  [9] repeat job key
25
+ [10] debounce key
25
26
  ARGV[2] Json stringified job data
26
27
  ARGV[3] msgpacked options
27
28
  Output:
@@ -39,8 +40,9 @@ local rcall = redis.call
39
40
  local args = cmsgpack.unpack(ARGV[1])
40
41
  local data = ARGV[2]
41
42
  local parentKey = args[5]
42
- local repeatJobKey = args[9]
43
43
  local parent = args[8]
44
+ local repeatJobKey = args[9]
45
+ local debounceKey = args[10]
44
46
  local parentData
45
47
  -- Includes
46
48
  --[[
@@ -67,6 +69,27 @@ local function addDelayMarkerIfNeeded(markerKey, delayedKey)
67
69
  rcall("ZADD", markerKey, nextTimestamp, "1")
68
70
  end
69
71
  end
72
+ --[[
73
+ Function to debounce a job.
74
+ ]]
75
+ local function debounceJob(prefixKey, debounceOpts, jobId, debounceKey, eventsKey, maxEvents)
76
+ local debounceId = debounceOpts and debounceOpts['id']
77
+ if debounceId then
78
+ local ttl = debounceOpts['ttl']
79
+ local debounceKeyExists
80
+ if ttl then
81
+ debounceKeyExists = not rcall('SET', debounceKey, jobId, 'PX', ttl, 'NX')
82
+ else
83
+ debounceKeyExists = not rcall('SET', debounceKey, jobId, 'NX')
84
+ end
85
+ if debounceKeyExists then
86
+ local currentDebounceJobId = rcall('GET', debounceKey)
87
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
88
+ "debounced", "jobId", currentDebounceJobId)
89
+ return currentDebounceJobId
90
+ end
91
+ end
92
+ end
70
93
  --[[
71
94
  Bake in the job id first 12 bits into the timestamp
72
95
  to guarantee correct execution order of delayed jobs
@@ -272,6 +295,7 @@ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
272
295
  local jsonOpts = cjson.encode(opts)
273
296
  local delay = opts['delay'] or 0
274
297
  local priority = opts['priority'] or 0
298
+ local debounceId = opts['de'] and opts['de']['id']
275
299
  local optionalValues = {}
276
300
  if parentKey ~= nil then
277
301
  table.insert(optionalValues, "parentKey")
@@ -283,6 +307,10 @@ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
283
307
  table.insert(optionalValues, "rjk")
284
308
  table.insert(optionalValues, repeatJobKey)
285
309
  end
310
+ if debounceId then
311
+ table.insert(optionalValues, "deid")
312
+ table.insert(optionalValues, debounceId)
313
+ end
286
314
  rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
287
315
  "timestamp", timestamp, "delay", delay, "priority", priority,
288
316
  unpack(optionalValues))
@@ -310,6 +338,11 @@ else
310
338
  maxEvents, timestamp)
311
339
  end
312
340
  end
341
+ local debouncedJobId = debounceJob(args[1], opts['de'],
342
+ jobId, debounceKey, eventsKey, maxEvents)
343
+ if debouncedJobId then
344
+ return debouncedJobId
345
+ end
313
346
  -- Store the job.
314
347
  local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
315
348
  opts, timestamp, parentKey, parentData,
@@ -1 +1 @@
1
- {"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwUf,CAAC;AACF,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyWf,CAAC;AACF,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -18,6 +18,7 @@ const content = `--[[
18
18
  [7] parent dependencies key.
19
19
  [8] parent? {id, queueKey}
20
20
  [9] repeat job key
21
+ [10] debounce key
21
22
  ARGV[2] Json stringified job data
22
23
  ARGV[3] msgpacked options
23
24
  Output:
@@ -35,10 +36,32 @@ local args = cmsgpack.unpack(ARGV[1])
35
36
  local data = ARGV[2]
36
37
  local opts = cmsgpack.unpack(ARGV[3])
37
38
  local parentKey = args[5]
38
- local repeatJobKey = args[9]
39
39
  local parent = args[8]
40
+ local repeatJobKey = args[9]
41
+ local debounceKey = args[10]
40
42
  local parentData
41
43
  -- Includes
44
+ --[[
45
+ Function to debounce a job.
46
+ ]]
47
+ local function debounceJob(prefixKey, debounceOpts, jobId, debounceKey, eventsKey, maxEvents)
48
+ local debounceId = debounceOpts and debounceOpts['id']
49
+ if debounceId then
50
+ local ttl = debounceOpts['ttl']
51
+ local debounceKeyExists
52
+ if ttl then
53
+ debounceKeyExists = not rcall('SET', debounceKey, jobId, 'PX', ttl, 'NX')
54
+ else
55
+ debounceKeyExists = not rcall('SET', debounceKey, jobId, 'NX')
56
+ end
57
+ if debounceKeyExists then
58
+ local currentDebounceJobId = rcall('GET', debounceKey)
59
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
60
+ "debounced", "jobId", currentDebounceJobId)
61
+ return currentDebounceJobId
62
+ end
63
+ end
64
+ end
42
65
  --[[
43
66
  Function to get max events value or set by default 10000.
44
67
  ]]
@@ -244,6 +267,7 @@ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
244
267
  local jsonOpts = cjson.encode(opts)
245
268
  local delay = opts['delay'] or 0
246
269
  local priority = opts['priority'] or 0
270
+ local debounceId = opts['de'] and opts['de']['id']
247
271
  local optionalValues = {}
248
272
  if parentKey ~= nil then
249
273
  table.insert(optionalValues, "parentKey")
@@ -255,6 +279,10 @@ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
255
279
  table.insert(optionalValues, "rjk")
256
280
  table.insert(optionalValues, repeatJobKey)
257
281
  end
282
+ if debounceId then
283
+ table.insert(optionalValues, "deid")
284
+ table.insert(optionalValues, debounceId)
285
+ end
258
286
  rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
259
287
  "timestamp", timestamp, "delay", delay, "priority", priority,
260
288
  unpack(optionalValues))
@@ -281,6 +309,11 @@ else
281
309
  maxEvents, timestamp)
282
310
  end
283
311
  end
312
+ local debouncedJobId = debounceJob(args[1], opts['de'],
313
+ jobId, debounceKey, eventsKey, maxEvents)
314
+ if debouncedJobId then
315
+ return debouncedJobId
316
+ end
284
317
  -- Store the job.
285
318
  storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2], opts, timestamp,
286
319
  parentKey, parentData, repeatJobKey)
@@ -1 +1 @@
1
- {"version":3,"file":"addParentJob-4.js","sourceRoot":"","sources":["../../../src/scripts/addParentJob-4.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAuSf,CAAC;AACF,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"addParentJob-4.js","sourceRoot":"","sources":["../../../src/scripts/addParentJob-4.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwUf,CAAC;AACF,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}