bullmq 4.16.0 → 5.0.0

package/dist/cjs/commands/{retryJob-9.lua → retryJob-10.lua}

@@ -2,15 +2,16 @@
   Retries a failed job by moving it back to the wait queue.
 
     Input:
-      KEYS[1] 'active',
-      KEYS[2] 'wait'
-      KEYS[3] 'paused'
-      KEYS[4] job key
-      KEYS[5] 'meta'
-      KEYS[6] events stream
-      KEYS[7] delayed key
-      KEYS[8] prioritized key
-      KEYS[9] 'pc' priority counter
+      KEYS[1]  'active',
+      KEYS[2]  'wait'
+      KEYS[3]  'paused'
+      KEYS[4]  job key
+      KEYS[5]  'meta'
+      KEYS[6]  events stream
+      KEYS[7]  delayed key
+      KEYS[8]  prioritized key
+      KEYS[9]  'pc' priority counter
+      KEYS[10] 'marker'
 
       ARGV[1]  key prefix
       ARGV[2]  timestamp
@@ -34,9 +35,11 @@ local rcall = redis.call
 --- @include "includes/promoteDelayedJobs"
 
 local target, paused = getTargetQueueList(KEYS[5], KEYS[2], KEYS[3])
+local markerKey = KEYS[10]
+
 -- Check if there are delayed jobs that we can move to wait.
 -- test example: when there are delayed jobs between retries
-promoteDelayedJobs(KEYS[7], KEYS[2], target, KEYS[8], KEYS[6], ARGV[1], ARGV[2], paused, KEYS[9])
+promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[8], KEYS[6], ARGV[1], ARGV[2], KEYS[9], paused)
 
 if rcall("EXISTS", KEYS[4]) == 1 then
 
@@ -57,9 +60,11 @@ if rcall("EXISTS", KEYS[4]) == 1 then
   if priority == 0 then
     rcall(ARGV[3], target, ARGV[4])
   else
-    addJobWithPriority(KEYS[2], KEYS[8], priority, paused, ARGV[4], KEYS[9])
+    addJobWithPriority(markerKey, KEYS[8], priority, ARGV[4], KEYS[9], paused)
   end
 
+  rcall("HINCRBY", KEYS[4], "atm", 1)
+
   local maxEvents = rcall("HGET", KEYS[5], "opts.maxLenEvents") or 10000
 
   -- Emit waiting event

package/dist/cjs/scripts/{addDelayedJob-7.js → addDelayedJob-6.js}

@@ -9,13 +9,12 @@ const content = `--[[
     - adds to delayed zset.
     - Emits a global event 'delayed' if the job is delayed.
     Input:
-      KEYS[1] 'wait',
-      KEYS[2] 'paused'
-      KEYS[3] 'meta'
-      KEYS[4] 'id'
-      KEYS[5] 'delayed'
-      KEYS[6] 'completed'
-      KEYS[7] events stream key
+      KEYS[1] 'marker',
+      KEYS[2] 'meta'
+      KEYS[3] 'id'
+      KEYS[4] 'delayed'
+      KEYS[5] 'completed'
+      KEYS[6] events stream key
       ARGV[1] msgpacked arguments array
             [1]  key prefix,
             [2]  custom id (use custom instead of one generated automatically)
@@ -32,13 +31,11 @@ const content = `--[[
         jobId  - OK
         -5     - Missing parent key
 ]]
-local waitKey = KEYS[1]
-local pausedKey = KEYS[2]
-local metaKey = KEYS[3]
-local idKey = KEYS[4]
-local delayedKey = KEYS[5]
-local completedKey = KEYS[6]
-local eventsKey = KEYS[7]
+local metaKey = KEYS[2]
+local idKey = KEYS[3]
+local delayedKey = KEYS[4]
+local completedKey = KEYS[5]
+local eventsKey = KEYS[6]
 local jobId
 local jobIdKey
 local rcall = redis.call
@@ -77,7 +74,7 @@ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
 end
 --[[
   Add delay marker if needed.
-]]
+]] 
 -- Includes
 --[[
   Function to return the next delayed job timestamp.
@@ -92,36 +89,20 @@ local function getNextDelayedTimestamp(delayedKey)
     return nextTimestamp
   end
 end
-local function addDelayMarkerIfNeeded(targetKey, delayedKey)
-  local waitLen = rcall("LLEN", targetKey)
-  if waitLen <= 1 then
+local function addDelayMarkerIfNeeded(markerKey, delayedKey)
     local nextTimestamp = getNextDelayedTimestamp(delayedKey)
     if nextTimestamp ~= nil then
-      -- Check if there is already a marker with older timestamp
-      -- if there is, we need to replace it.
-      if waitLen == 1 then
-        local marker = rcall("LINDEX", targetKey, 0)
-        local oldTimestamp = tonumber(marker:sub(3))
-        if oldTimestamp and oldTimestamp > nextTimestamp then
-          rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
-        end
-      else
-        -- if there is no marker, then we need to add one
-        rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
-      end
+        -- Replace the score of the marker with the newest known
+        -- next timestamp.
+        rcall("ZADD", markerKey, nextTimestamp, "0")
     end
-  end
 end
 --[[
   Function to check for the meta.paused key to decide if we are paused or not
   (since an empty list and !EXISTS are not really the same).
 ]]
-local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
-  if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
-    return waitKey, false
-  else
-    return pausedKey, true
-  end
+local function isQueuePaused(queueMetaKey)
+    return rcall("HEXISTS", queueMetaKey, "paused") == 1
 end
 --[[
   Validate and move or add dependencies to parent.
@@ -134,56 +115,64 @@ end
 --[[
   Function to add job considering priority.
 ]]
--- Includes
---[[
-  Function priority marker to wait if needed
-  in order to wake up our workers and to respect priority
-  order as much as possible
-]]
-local function addPriorityMarkerIfNeeded(waitKey)
-  local waitLen = rcall("LLEN", waitKey)
-  if waitLen == 0 then
-    rcall("LPUSH", waitKey, "0:0")
-  end
-end
-local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
+local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
   local prioCounter = rcall("INCR", priorityCounterKey)
   local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
   rcall("ZADD", prioritizedKey, score, jobId)
-  if not paused then
-    addPriorityMarkerIfNeeded(waitKey)
+  if not isPaused then
+    rcall("ZADD", markerKey, 0, "0")
   end
 end
-local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
-  local isParentActive = rcall("ZSCORE", parentQueueKey .. ":waiting-children", parentId)
-  if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then 
-    rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
-    local parentWaitKey = parentQueueKey .. ":wait"
-    local parentTarget, paused = getTargetQueueList(parentQueueKey .. ":meta", parentWaitKey,
-      parentQueueKey .. ":paused")
-    local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
-    local priority = tonumber(jobAttributes[1]) or 0
-    local delay = tonumber(jobAttributes[2]) or 0
-    if delay > 0 then
-      local delayedTimestamp = tonumber(timestamp) + delay 
-      local score = delayedTimestamp * 0x1000
-      local parentDelayedKey = parentQueueKey .. ":delayed" 
-      rcall("ZADD", parentDelayedKey, score, parentId)
-      rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId,
-        "delay", delayedTimestamp)
-      addDelayMarkerIfNeeded(parentTarget, parentDelayedKey)
-    else
-      if priority == 0 then
-        rcall("RPUSH", parentTarget, parentId)
-      else
-        addJobWithPriority(parentWaitKey, parentQueueKey .. ":prioritized", priority, paused,
-          parentId, parentQueueKey .. ":pc")
-      end
-      rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId,
-        "prev", "waiting-children")
-    end
+--[[
+  Function to check for the meta.paused key to decide if we are paused or not
+  (since an empty list and !EXISTS are not really the same).
+]]
+local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
+  if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
+    return waitKey, false
+  else
+    return pausedKey, true
   end
 end
+local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
+                                        parentKey, parentId, timestamp)
+    local isParentActive = rcall("ZSCORE",
+                                 parentQueueKey .. ":waiting-children", parentId)
+    if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
+        rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
+        local parentWaitKey = parentQueueKey .. ":wait"
+        local parentPausedKey = parentQueueKey .. ":paused"
+        local parentMetaKey = parentQueueKey .. ":meta"
+        local parentMarkerKey = parentQueueKey .. ":marker"
+        local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
+        local priority = tonumber(jobAttributes[1]) or 0
+        local delay = tonumber(jobAttributes[2]) or 0
+        if delay > 0 then
+            local delayedTimestamp = tonumber(timestamp) + delay
+            local score = delayedTimestamp * 0x1000
+            local parentDelayedKey = parentQueueKey .. ":delayed"
+            rcall("ZADD", parentDelayedKey, score, parentId)
+            rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
+                  "jobId", parentId, "delay", delayedTimestamp)
+            addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
+        else
+            if priority == 0 then
+                local parentTarget, _paused =
+                    getTargetQueueList(parentMetaKey, parentWaitKey,
+                                       parentPausedKey)
+                rcall("RPUSH", parentTarget, parentId)
+                rcall("ZADD", parentMarkerKey, 0, "0")
+            else
+                local isPaused = isQueuePaused(parentMetaKey)
+                addJobWithPriority(parentMarkerKey,
+                                   parentQueueKey .. ":prioritized", priority,
+                                   parentId, parentQueueKey .. ":pc", isPaused)
+            end
+            rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
+                  "jobId", parentId, "prev", "waiting-children")
+        end
+    end
+end
 local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
   parentId, jobIdKey, returnvalue, timestamp )
   local processedSet = parentKey .. ":processed"
@@ -255,13 +244,13 @@ local score = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff)
 rcall("ZADD", delayedKey, score, jobId)
 rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
       "jobId", jobId, "delay", delayedTimestamp)
--- If wait list is empty, and this delayed job is the next one to be processed,
--- then we need to signal the workers by adding a dummy job (jobId 0:delay) to the wait list.
-local target = getTargetQueueList(metaKey, KEYS[1], KEYS[2])
-addDelayMarkerIfNeeded(target, delayedKey)
+-- mark that a delayed job is available
+local isPaused = isQueuePaused(metaKey)
+if not isPaused then
+    local markerKey = KEYS[1]
+    addDelayMarkerIfNeeded(markerKey, delayedKey)
+end
 -- Check if this job is a child of another job, if so add it to the parents dependencies
--- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
--- fail in this case.
 if parentDependenciesKey ~= nil then
     rcall("SADD", parentDependenciesKey, jobIdKey)
 end
@@ -270,6 +259,6 @@ return jobId .. "" -- convert to string
 exports.addDelayedJob = {
     name: 'addDelayedJob',
     content,
-    keys: 7,
+    keys: 6,
 };
-//# sourceMappingURL=addDelayedJob-7.js.map
+//# sourceMappingURL=addDelayedJob-6.js.map

package/dist/cjs/scripts/{moveToActive-10.js.map → addDelayedJob-6.js.map}

@@ -1 +1 @@
-{"version":3,"file":"moveToActive-10.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-10.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAmPf,CAAC;AACW,QAAA,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
+{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA8Pf,CAAC;AACW,QAAA,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}

package/dist/cjs/scripts/addParentJob-4.js

@@ -77,7 +77,7 @@ end
 -- Includes
 --[[
   Add delay marker if needed.
-]]
+]] 
 -- Includes
 --[[
   Function to return the next delayed job timestamp.
@@ -92,47 +92,30 @@ local function getNextDelayedTimestamp(delayedKey)
     return nextTimestamp
   end
 end
-local function addDelayMarkerIfNeeded(targetKey, delayedKey)
-  local waitLen = rcall("LLEN", targetKey)
-  if waitLen <= 1 then
+local function addDelayMarkerIfNeeded(markerKey, delayedKey)
     local nextTimestamp = getNextDelayedTimestamp(delayedKey)
     if nextTimestamp ~= nil then
-      -- Check if there is already a marker with older timestamp
-      -- if there is, we need to replace it.
-      if waitLen == 1 then
-        local marker = rcall("LINDEX", targetKey, 0)
-        local oldTimestamp = tonumber(marker:sub(3))
-        if oldTimestamp and oldTimestamp > nextTimestamp then
-          rcall("LSET", targetKey, 0, "0:" .. nextTimestamp)
-        end
-      else
-        -- if there is no marker, then we need to add one
-        rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
-      end
+        -- Replace the score of the marker with the newest known
+        -- next timestamp.
+        rcall("ZADD", markerKey, nextTimestamp, "0")
     end
-  end
 end
 --[[
-  Function to add job considering priority.
+  Function to check for the meta.paused key to decide if we are paused or not
+  (since an empty list and !EXISTS are not really the same).
 ]]
--- Includes
+local function isQueuePaused(queueMetaKey)
+    return rcall("HEXISTS", queueMetaKey, "paused") == 1
+end
 --[[
-  Function priority marker to wait if needed
-  in order to wake up our workers and to respect priority
-  order as much as possible
+  Function to add job considering priority.
 ]]
-local function addPriorityMarkerIfNeeded(waitKey)
-  local waitLen = rcall("LLEN", waitKey)
-  if waitLen == 0 then
-    rcall("LPUSH", waitKey, "0:0")
-  end
-end
-local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
+local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
   local prioCounter = rcall("INCR", priorityCounterKey)
   local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
   rcall("ZADD", prioritizedKey, score, jobId)
-  if not paused then
-    addPriorityMarkerIfNeeded(waitKey)
+  if not isPaused then
+    rcall("ZADD", markerKey, 0, "0")
   end
 end
 --[[
@@ -146,35 +129,44 @@ local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
     return pausedKey, true
   end
 end
-local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
-  local isParentActive = rcall("ZSCORE", parentQueueKey .. ":waiting-children", parentId)
-  if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then 
-    rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
-    local parentWaitKey = parentQueueKey .. ":wait"
-    local parentTarget, paused = getTargetQueueList(parentQueueKey .. ":meta", parentWaitKey,
-      parentQueueKey .. ":paused")
-    local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
-    local priority = tonumber(jobAttributes[1]) or 0
-    local delay = tonumber(jobAttributes[2]) or 0
-    if delay > 0 then
-      local delayedTimestamp = tonumber(timestamp) + delay 
-      local score = delayedTimestamp * 0x1000
-      local parentDelayedKey = parentQueueKey .. ":delayed" 
-      rcall("ZADD", parentDelayedKey, score, parentId)
-      rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId,
-        "delay", delayedTimestamp)
-      addDelayMarkerIfNeeded(parentTarget, parentDelayedKey)
-    else
-      if priority == 0 then
-        rcall("RPUSH", parentTarget, parentId)
-      else
-        addJobWithPriority(parentWaitKey, parentQueueKey .. ":prioritized", priority, paused,
-          parentId, parentQueueKey .. ":pc")
-      end
-      rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId,
-        "prev", "waiting-children")
+local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
+                                        parentKey, parentId, timestamp)
+    local isParentActive = rcall("ZSCORE",
+                                 parentQueueKey .. ":waiting-children", parentId)
+    if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
+        rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
+        local parentWaitKey = parentQueueKey .. ":wait"
+        local parentPausedKey = parentQueueKey .. ":paused"
+        local parentMetaKey = parentQueueKey .. ":meta"
+        local parentMarkerKey = parentQueueKey .. ":marker"
+        local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
+        local priority = tonumber(jobAttributes[1]) or 0
+        local delay = tonumber(jobAttributes[2]) or 0
+        if delay > 0 then
+            local delayedTimestamp = tonumber(timestamp) + delay
+            local score = delayedTimestamp * 0x1000
+            local parentDelayedKey = parentQueueKey .. ":delayed"
+            rcall("ZADD", parentDelayedKey, score, parentId)
+            rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
+                  "jobId", parentId, "delay", delayedTimestamp)
+            addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
+        else
+            if priority == 0 then
+                local parentTarget, _paused =
+                    getTargetQueueList(parentMetaKey, parentWaitKey,
+                                       parentPausedKey)
+                rcall("RPUSH", parentTarget, parentId)
+                rcall("ZADD", parentMarkerKey, 0, "0")
+            else
+                local isPaused = isQueuePaused(parentMetaKey)
+                addJobWithPriority(parentMarkerKey,
+                                   parentQueueKey .. ":prioritized", priority,
+                                   parentId, parentQueueKey .. ":pc", isPaused)
+            end
+            rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
+                  "jobId", parentId, "prev", "waiting-children")
+        end
     end
-  end
 end
 local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
   parentId, jobIdKey, returnvalue, timestamp )
@@ -244,8 +236,6 @@ rcall("ZADD", waitChildrenKey, timestamp, jobId)
 rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
       "waiting-children", "jobId", jobId)
 -- Check if this job is a child of another job, if so add it to the parents dependencies
--- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
--- fail in this case.
 if parentDependenciesKey ~= nil then
     rcall("SADD", parentDependenciesKey, jobIdKey)
 end

package/dist/cjs/scripts/addParentJob-4.js.map

@@ -1 +1 @@
-{"version":3,"file":"addParentJob-4.js","sourceRoot":"","sources":["../../../src/scripts/addParentJob-4.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyPf,CAAC;AACW,QAAA,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
+{"version":3,"file":"addParentJob-4.js","sourceRoot":"","sources":["../../../src/scripts/addParentJob-4.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+Of,CAAC;AACW,QAAA,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}