bullmq 5.8.7 → 5.9.0

package/dist/esm/scripts/removeChildDependency-1.js

@@ -28,14 +28,14 @@ local parentKey = ARGV[2]
 --[[
   Add marker if needed when a job is available.
 ]]
-local function addBaseMarkerIfNeeded(markerKey, isPaused)
-  if not isPaused then
+local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
+  if not isPausedOrMaxed then
     rcall("ZADD", markerKey, 0, "0")
   end  
 end
-local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
+local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
   rcall(pushCmd, targetKey, jobId)
-  addBaseMarkerIfNeeded(markerKey, isPaused)
+  addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
 end
 --[[
   Functions to destructure job key.
@@ -51,12 +51,21 @@ end
   Function to check for the meta.paused key to decide if we are paused or not
   (since an empty list and !EXISTS are not really the same).
 ]]
-local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
-  if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
-    return waitKey, false
-  else
+local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
+  local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
+  if queueAttributes[1] then
     return pausedKey, true
+  else
+    if queueAttributes[2] then
+      local activeCount = rcall("LLEN", activeKey)
+      if activeCount >= tonumber(queueAttributes[2]) then
+        return waitKey, true
+      else
+        return waitKey, false
+      end
+    end
   end
+  return waitKey, false
 end
 --[[
   Function to remove job keys.
@@ -66,9 +75,9 @@ local function removeJobKeys(jobKey)
     jobKey .. ':dependencies', jobKey .. ':processed', jobKey .. ':failed')
 end
 local function moveParentToWait(parentPrefix, parentId, emitEvent)
-  local parentTarget, isPaused = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "wait",
-    parentPrefix .. "paused")
-  addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPaused, parentId)
+  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
+    parentPrefix .. "wait", parentPrefix .. "paused")
+  addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
   if emitEvent then
     local parentEventStream = parentPrefix .. "events"
     rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")

package/dist/esm/scripts/removeChildDependency-1.js.map

@@ -1 +1 @@
-{"version":3,"file":"removeChildDependency-1.js","sourceRoot":"","sources":["../../../src/scripts/removeChildDependency-1.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IA2IZ,CAAC;AACL,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
+{"version":3,"file":"removeChildDependency-1.js","sourceRoot":"","sources":["../../../src/scripts/removeChildDependency-1.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;IAoJZ,CAAC;AACL,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}

package/dist/esm/scripts/{removeJob-1.js → removeJob-2.js}

@@ -3,6 +3,7 @@ const content = `--[[
     In order to be able to remove a job, it cannot be active.
     Input:
       KEYS[1] queue prefix
+      KEYS[2] meta key
       ARGV[1] jobId
       ARGV[2] remove children
     Events:
@@ -113,30 +114,39 @@ end
 --[[
   Add marker if needed when a job is available.
 ]]
-local function addBaseMarkerIfNeeded(markerKey, isPaused)
-  if not isPaused then
+local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
+  if not isPausedOrMaxed then
     rcall("ZADD", markerKey, 0, "0")
   end  
 end
-local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
+local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
   rcall(pushCmd, targetKey, jobId)
-  addBaseMarkerIfNeeded(markerKey, isPaused)
+  addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
 end
 --[[
   Function to check for the meta.paused key to decide if we are paused or not
   (since an empty list and !EXISTS are not really the same).
 ]]
-local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
-  if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
-    return waitKey, false
-  else
+local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
+  local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
+  if queueAttributes[1] then
     return pausedKey, true
+  else
+    if queueAttributes[2] then
+      local activeCount = rcall("LLEN", activeKey)
+      if activeCount >= tonumber(queueAttributes[2]) then
+        return waitKey, true
+      else
+        return waitKey, false
+      end
+    end
   end
+  return waitKey, false
 end
 local function moveParentToWait(parentPrefix, parentId, emitEvent)
-  local parentTarget, isPaused = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "wait",
-    parentPrefix .. "paused")
-  addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPaused, parentId)
+  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
+    parentPrefix .. "wait", parentPrefix .. "paused")
+  addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
   if emitEvent then
     local parentEventStream = parentPrefix .. "events"
     rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
@@ -234,7 +244,7 @@ local function removeJob( prefix, jobId, parentKey, removeChildren)
     end
     local prev = removeJobFromAnyState(prefix, jobId)
     if removeJobKeys(jobKey) > 0 then
-        local maxEvents = getOrSetMaxEvents(prefix .. "meta")
+        local maxEvents = getOrSetMaxEvents(KEYS[2])
         rcall("XADD", prefix .. "events", "MAXLEN", "~", maxEvents, "*", "event", "removed",
             "jobId", jobId, "prev", prev)
     end
@@ -249,6 +259,6 @@ return 0
 export const removeJob = {
     name: 'removeJob',
     content,
-    keys: 1,
+    keys: 2,
 };
-//# sourceMappingURL=removeJob-1.js.map
+//# sourceMappingURL=removeJob-2.js.map

package/dist/esm/scripts/{removeJob-1.js.map → removeJob-2.js.map}

@@ -1 +1 @@
-{"version":3,"file":"removeJob-1.js","sourceRoot":"","sources":["../../../src/scripts/removeJob-1.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAuPf,CAAC;AACF,MAAM,CAAC,MAAM,SAAS,GAAG;IACvB,IAAI,EAAE,WAAW;IACjB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
+{"version":3,"file":"removeJob-2.js","sourceRoot":"","sources":["../../../src/scripts/removeJob-2.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAiQf,CAAC;AACF,MAAM,CAAC,MAAM,SAAS,GAAG;IACvB,IAAI,EAAE,WAAW;IACjB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}

package/dist/esm/scripts/{reprocessJob-7.js → reprocessJob-8.js}

@@ -7,7 +7,8 @@ const content = `--[[
     KEYS[4] wait key
     KEYS[5] meta
     KEYS[6] paused key
-    KEYS[7] marker key
+    KEYS[7] active key
+    KEYS[8] marker key
     ARGV[1] job.id
     ARGV[2] (job.opts.lifo ? 'R' : 'L') + 'PUSH'
     ARGV[3] propVal - failedReason/returnvalue
@@ -26,14 +27,14 @@ local rcall = redis.call;
 --[[
   Add marker if needed when a job is available.
 ]]
-local function addBaseMarkerIfNeeded(markerKey, isPaused)
-  if not isPaused then
+local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
+  if not isPausedOrMaxed then
     rcall("ZADD", markerKey, 0, "0")
   end  
 end
-local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
+local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
   rcall(pushCmd, targetKey, jobId)
-  addBaseMarkerIfNeeded(markerKey, isPaused)
+  addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
 end
 --[[
   Function to get max events value or set by default 10000.
@@ -50,19 +51,28 @@ end
   Function to check for the meta.paused key to decide if we are paused or not
   (since an empty list and !EXISTS are not really the same).
 ]]
-local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
-  if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
-    return waitKey, false
-  else
+local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
+  local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
+  if queueAttributes[1] then
     return pausedKey, true
+  else
+    if queueAttributes[2] then
+      local activeCount = rcall("LLEN", activeKey)
+      if activeCount >= tonumber(queueAttributes[2]) then
+        return waitKey, true
+      else
+        return waitKey, false
+      end
+    end
   end
+  return waitKey, false
 end
 if rcall("EXISTS", KEYS[1]) == 1 then
   local jobId = ARGV[1]
   if (rcall("ZREM", KEYS[3], jobId) == 1) then
     rcall("HDEL", KEYS[1], "finishedOn", "processedOn", ARGV[3])
-    local target, isPaused = getTargetQueueList(KEYS[5], KEYS[4], KEYS[6])
-    addJobInTargetList(target, KEYS[7], ARGV[2], isPaused, jobId)
+    local target, isPausedOrMaxed = getTargetQueueList(KEYS[5], KEYS[7], KEYS[4], KEYS[6])
+    addJobInTargetList(target, KEYS[8], ARGV[2], isPausedOrMaxed, jobId)
     local maxEvents = getOrSetMaxEvents(KEYS[5])
     -- Emit waiting event
     rcall("XADD", KEYS[2], "MAXLEN", "~", maxEvents, "*", "event", "waiting",
@@ -78,6 +88,6 @@ end
 export const reprocessJob = {
     name: 'reprocessJob',
     content,
-    keys: 7,
+    keys: 8,
 };
-//# sourceMappingURL=reprocessJob-7.js.map
+//# sourceMappingURL=reprocessJob-8.js.map

package/dist/esm/scripts/reprocessJob-8.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"reprocessJob-8.js","sourceRoot":"","sources":["../../../src/scripts/reprocessJob-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsFf,CAAC;AACF,MAAM,CAAC,MAAM,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}

package/dist/esm/scripts/retryJob-11.js

@@ -34,24 +34,25 @@ local rcall = redis.call
 --[[
   Add marker if needed when a job is available.
 ]]
-local function addBaseMarkerIfNeeded(markerKey, isPaused)
-  if not isPaused then
+local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
+  if not isPausedOrMaxed then
     rcall("ZADD", markerKey, 0, "0")
   end  
 end
-local function addJobInTargetList(targetKey, markerKey, pushCmd, isPaused, jobId)
+local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
   rcall(pushCmd, targetKey, jobId)
-  addBaseMarkerIfNeeded(markerKey, isPaused)
+  addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
 end
 --[[
   Function to add job considering priority.
 ]]
 -- Includes
-local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey, isPaused)
+local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
+  isPausedOrMaxed)
   local prioCounter = rcall("INCR", priorityCounterKey)
   local score = priority * 0x100000000 + prioCounter % 0x100000000
   rcall("ZADD", prioritizedKey, score, jobId)
-  addBaseMarkerIfNeeded(markerKey, isPaused)
+  addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
 end
 --[[
   Function to get max events value or set by default 10000.
@@ -68,12 +69,21 @@ end
   Function to check for the meta.paused key to decide if we are paused or not
   (since an empty list and !EXISTS are not really the same).
 ]]
-local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
-  if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
-    return waitKey, false
-  else
+local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
+  local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
+  if queueAttributes[1] then
     return pausedKey, true
+  else
+    if queueAttributes[2] then
+      local activeCount = rcall("LLEN", activeKey)
+      if activeCount >= tonumber(queueAttributes[2]) then
+        return waitKey, true
+      else
+        return waitKey, false
+      end
+    end
   end
+  return waitKey, false
 end
 --[[
   Updates the delay set, by moving delayed jobs that should
@@ -125,11 +135,27 @@ local function removeLock(jobKey, stalledKey, token, jobId)
   end
   return 0
 end
-local target, paused = getTargetQueueList(KEYS[5], KEYS[2], KEYS[3])
+--[[
+  Function to check if queue is paused or maxed
+  (since an empty list and !EXISTS are not really the same).
+]]
+local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
+  local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
+  if queueAttributes[1] then
+    return true
+  else
+    if queueAttributes[2] then
+      local activeCount = rcall("LLEN", activeKey)
+      return activeCount >= tonumber(queueAttributes[2])
+    end
+  end
+  return false
+end
+local target, isPausedOrMaxed = getTargetQueueList(KEYS[5], KEYS[1], KEYS[2], KEYS[3])
 local markerKey = KEYS[10]
 -- Check if there are delayed jobs that we can move to wait.
 -- test example: when there are delayed jobs between retries
-promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[8], KEYS[6], ARGV[1], ARGV[2], KEYS[9], paused)
+promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[8], KEYS[6], ARGV[1], ARGV[2], KEYS[9], isPausedOrMaxed)
 if rcall("EXISTS", KEYS[4]) == 1 then
   local errorCode = removeLock(KEYS[4], KEYS[11], ARGV[5], ARGV[4]) 
   if errorCode < 0 then
@@ -138,11 +164,13 @@ if rcall("EXISTS", KEYS[4]) == 1 then
   local numRemovedElements = rcall("LREM", KEYS[1], -1, ARGV[4])
   if (numRemovedElements < 1) then return -3 end
   local priority = tonumber(rcall("HGET", KEYS[4], "priority")) or 0
+  --need to re-evaluate after removing job from active
+  isPausedOrMaxed = isQueuePausedOrMaxed(KEYS[5], KEYS[1])
   -- Standard or priority add
   if priority == 0 then
-    addJobInTargetList(target, KEYS[10], ARGV[3], paused, ARGV[4])
+    addJobInTargetList(target, markerKey, ARGV[3], isPausedOrMaxed, ARGV[4])
   else
-    addJobWithPriority(markerKey, KEYS[8], priority, ARGV[4], KEYS[9], paused)
+    addJobWithPriority(markerKey, KEYS[8], priority, ARGV[4], KEYS[9], isPausedOrMaxed)
   end
   rcall("HINCRBY", KEYS[4], "atm", 1)
   local maxEvents = getOrSetMaxEvents(KEYS[5])

package/dist/esm/scripts/retryJob-11.js.map

@@ -1 +1 @@
-{"version":3,"file":"retryJob-11.js","sourceRoot":"","sources":["../../../src/scripts/retryJob-11.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2Jf,CAAC;AACF,MAAM,CAAC,MAAM,QAAQ,GAAG;IACtB,IAAI,EAAE,UAAU;IAChB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
+{"version":3,"file":"retryJob-11.js","sourceRoot":"","sources":["../../../src/scripts/retryJob-11.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAuLf,CAAC;AACF,MAAM,CAAC,MAAM,QAAQ,GAAG;IACtB,IAAI,EAAE,UAAU;IAChB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}