bullmq 5.30.0 → 5.31.0

package/dist/cjs/scripts/moveToFinished-14.js

@@ -278,117 +278,8 @@ local function removeDeduplicationKeyIfNeeded(prefixKey, deduplicationId)
     end
   end
 end
-local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
-  if rcall("ZREM", parentQueueKey .. ":waiting-children", parentId) == 1 then
-    rcall("ZADD", parentQueueKey .. ":failed", timestamp, parentId)
-    local failedReason = "child " .. jobIdKey .. " failed"
-    rcall("HMSET", parentKey, "failedReason", failedReason, "finishedOn", timestamp)
-    rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason",
-      failedReason, "prev", "waiting-children")
-    local jobAttributes = rcall("HMGET", parentKey, "parent", "deid")
-    removeDeduplicationKeyIfNeeded(parentQueueKey .. ":", jobAttributes[2])
-    if jobAttributes[1] then
-      local parentData = cjson.decode(jobAttributes[1])
-      if parentData['fpof'] then
-        moveParentFromWaitingChildrenToFailed(
-          parentData['queueKey'],
-          parentData['queueKey'] .. ':' .. parentData['id'],
-          parentData['id'],
-          parentKey,
-          timestamp
-        )
-      elseif parentData['idof'] or parentData['rdof'] then
-        local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
-        local grandParentDependenciesSet = grandParentKey .. ":dependencies"
-        if rcall("SREM", grandParentDependenciesSet, parentKey) == 1 then
-          moveParentToWaitIfNeeded(parentData['queueKey'], grandParentDependenciesSet,
-            grandParentKey, parentData['id'], timestamp)
-          if parentData['idof'] then
-            local grandParentFailedSet = grandParentKey .. ":failed"
-            rcall("HSET", grandParentFailedSet, parentKey, failedReason)
-          end
-        end
-      end
-    end
-  end
-end
---[[
-  Function to move job from wait state to active.
-  Input:
-    opts - token - lock token
-    opts - lockDuration
-    opts - limiter
-]]
--- Includes
-local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey,
-    jobId, processedOn, maxJobs, markerKey, opts)
-  local jobKey = keyPrefix .. jobId
-  -- Check if we need to perform rate limiting.
-  if maxJobs then
-    local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
-    if jobCounter == 1 then
-      local limiterDuration = opts['limiter'] and opts['limiter']['duration']
-      local integerDuration = math.floor(math.abs(limiterDuration))
-      rcall("PEXPIRE", rateLimiterKey, integerDuration)
-    end
-  end
-  local lockKey = jobKey .. ':lock'
-  -- get a lock
-  if opts['token'] ~= "0" then
-    rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
-  end
-  local optionalValues = {}
-  if opts['name'] then
-    -- Set "processedBy" field to the worker name
-    table.insert(optionalValues, "pb")
-    table.insert(optionalValues, opts['name'])
-  end
-  rcall("XADD", eventStreamKey, "*", "event", "active", "jobId", jobId, "prev", "waiting")
-  rcall("HMSET", jobKey, "processedOn", processedOn, unpack(optionalValues))
-  rcall("HINCRBY", jobKey, "ats", 1)
-  addBaseMarkerIfNeeded(markerKey, false)
-  return {rcall("HGETALL", jobKey), jobId, 0, 0} -- get job data
-end
---[[
-  Updates the delay set, by moving delayed jobs that should
-  be processed now to "wait".
-     Events:
-      'waiting'
-]]
--- Includes
--- Try to get as much as 1000 jobs at once
-local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedKey,
-                                  eventStreamKey, prefix, timestamp, priorityCounterKey, isPaused)
-    local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000 - 1, "LIMIT", 0, 1000)
-    if (#jobs > 0) then
-        rcall("ZREM", delayedKey, unpack(jobs))
-        for _, jobId in ipairs(jobs) do
-            local jobKey = prefix .. jobId
-            local priority =
-                tonumber(rcall("HGET", jobKey, "priority")) or 0
-            if priority == 0 then
-                -- LIFO or FIFO
-                addJobInTargetList(targetKey, markerKey, "LPUSH", isPaused, jobId)
-            else
-                addJobWithPriority(markerKey, prioritizedKey, priority,
-                  jobId, priorityCounterKey, isPaused)
-            end
-            -- Emit waiting event
-            rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
-                  jobId, "prev", "delayed")
-            rcall("HSET", jobKey, "delay", 0)
-        end
-    end
-end
---[[
-  Function to remove job keys.
-]]
-local function removeJobKeys(jobKey)
-  return rcall("DEL", jobKey, jobKey .. ':logs',
-    jobKey .. ':dependencies', jobKey .. ':processed', jobKey .. ':failed')
-end
 --[[
-  Functions to remove jobs by max age.
+  Functions to remove jobs when removeOnFail option is provided.
 ]]
 -- Includes
 --[[
@@ -405,6 +296,13 @@ local function removeDeduplicationKey(prefixKey, jobKey)
     rcall("DEL", deduplicationKey)
   end
 end
+--[[
+  Function to remove job keys.
+]]
+local function removeJobKeys(jobKey)
+  return rcall("DEL", jobKey, jobKey .. ':logs',
+    jobKey .. ':dependencies', jobKey .. ':processed', jobKey .. ':failed')
+end
 --[[
   Check if this job has a parent. If so we will just remove it from
   the parent child list, but if it is the last child we should move the parent to "wait/paused"
@@ -501,6 +399,10 @@ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
   end
   removeJobKeys(jobKey)
 end
+--[[
+  Functions to remove jobs by max age.
+]]
+-- Includes
 local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix,
   shouldRemoveDebounceKey)
   local start = timestamp - maxAge * 1000
@@ -522,6 +424,137 @@ local function removeJobsByMaxCount(maxCount, targetSet, prefix)
   end
   rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
 end
+local function removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestamp)
+  local removeOnFailType = type(opts["removeOnFail"])
+  if removeOnFailType == "number" then
+    removeJobsByMaxCount(opts["removeOnFail"],
+                        failedKey, queueKeyPrefix)
+  elseif removeOnFailType == "boolean" then
+    if opts["removeOnFail"] then
+      removeJob(jobId, false, queueKeyPrefix,
+                false --[[remove debounce key]])
+      rcall("ZREM", failedKey, jobId)
+    end
+  elseif removeOnFailType ~= "nil" then
+    local maxAge = opts["removeOnFail"]["age"]
+    local maxCount = opts["removeOnFail"]["count"]
+    if maxAge ~= nil then
+      removeJobsByMaxAge(timestamp, maxAge,
+                        failedKey, queueKeyPrefix)
+    end
+    if maxCount ~= nil and maxCount > 0 then
+      removeJobsByMaxCount(maxCount, failedKey,
+                            queueKeyPrefix)
+    end
+  end 
+end
+local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
+  if rcall("ZREM", parentQueueKey .. ":waiting-children", parentId) == 1 then
+    local parentQueuePrefix = parentQueueKey .. ":"
+    local parentFailedKey = parentQueueKey .. ":failed"
+    rcall("ZADD", parentFailedKey, timestamp, parentId)
+    local failedReason = "child " .. jobIdKey .. " failed"
+    rcall("HMSET", parentKey, "failedReason", failedReason, "finishedOn", timestamp)
+    rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason",
+      failedReason, "prev", "waiting-children")
+    local jobAttributes = rcall("HMGET", parentKey, "parent", "deid", "opts")
+    removeDeduplicationKeyIfNeeded(parentQueueKey .. ":", jobAttributes[2])
+    if jobAttributes[1] then
+      local parentData = cjson.decode(jobAttributes[1])
+      if parentData['fpof'] then
+        moveParentFromWaitingChildrenToFailed(
+          parentData['queueKey'],
+          parentData['queueKey'] .. ':' .. parentData['id'],
+          parentData['id'],
+          parentKey,
+          timestamp
+        )
+      elseif parentData['idof'] or parentData['rdof'] then
+        local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
+        local grandParentDependenciesSet = grandParentKey .. ":dependencies"
+        if rcall("SREM", grandParentDependenciesSet, parentKey) == 1 then
+          moveParentToWaitIfNeeded(parentData['queueKey'], grandParentDependenciesSet,
+            grandParentKey, parentData['id'], timestamp)
+          if parentData['idof'] then
+            local grandParentFailedSet = grandParentKey .. ":failed"
+            rcall("HSET", grandParentFailedSet, parentKey, failedReason)
+          end
+        end
+      end
+    end
+    local parentRawOpts = jobAttributes[3]
+    local parentOpts = cjson.decode(parentRawOpts)
+    removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
+  end
+end
+--[[
+  Function to move job from wait state to active.
+  Input:
+    opts - token - lock token
+    opts - lockDuration
+    opts - limiter
+]]
+-- Includes
+local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey,
+    jobId, processedOn, maxJobs, markerKey, opts)
+  local jobKey = keyPrefix .. jobId
+  -- Check if we need to perform rate limiting.
+  if maxJobs then
+    local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
+    if jobCounter == 1 then
+      local limiterDuration = opts['limiter'] and opts['limiter']['duration']
+      local integerDuration = math.floor(math.abs(limiterDuration))
+      rcall("PEXPIRE", rateLimiterKey, integerDuration)
+    end
+  end
+  local lockKey = jobKey .. ':lock'
+  -- get a lock
+  if opts['token'] ~= "0" then
+    rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
+  end
+  local optionalValues = {}
+  if opts['name'] then
+    -- Set "processedBy" field to the worker name
+    table.insert(optionalValues, "pb")
+    table.insert(optionalValues, opts['name'])
+  end
+  rcall("XADD", eventStreamKey, "*", "event", "active", "jobId", jobId, "prev", "waiting")
+  rcall("HMSET", jobKey, "processedOn", processedOn, unpack(optionalValues))
+  rcall("HINCRBY", jobKey, "ats", 1)
+  addBaseMarkerIfNeeded(markerKey, false)
+  return {rcall("HGETALL", jobKey), jobId, 0, 0} -- get job data
+end
+--[[
+  Updates the delay set, by moving delayed jobs that should
+  be processed now to "wait".
+     Events:
+      'waiting'
+]]
+-- Includes
+-- Try to get as much as 1000 jobs at once
+local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedKey,
+                                  eventStreamKey, prefix, timestamp, priorityCounterKey, isPaused)
+    local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000 - 1, "LIMIT", 0, 1000)
+    if (#jobs > 0) then
+        rcall("ZREM", delayedKey, unpack(jobs))
+        for _, jobId in ipairs(jobs) do
+            local jobKey = prefix .. jobId
+            local priority =
+                tonumber(rcall("HGET", jobKey, "priority")) or 0
+            if priority == 0 then
+                -- LIFO or FIFO
+                addJobInTargetList(targetKey, markerKey, "LPUSH", isPaused, jobId)
+            else
+                addJobWithPriority(markerKey, prioritizedKey, priority,
+                  jobId, priorityCounterKey, isPaused)
+            end
+            -- Emit waiting event
+            rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
+                  jobId, "prev", "delayed")
+            rcall("HSET", jobKey, "delay", 0)
+        end
+    end
+end
 local function removeLock(jobKey, stalledKey, token, jobId)
   if token ~= "0" then
     local lockKey = jobKey .. ':lock'

package/dist/cjs/scripts/moveToFinished-14.js.map

@@ -1 +1 @@
-{"version":3,"file":"moveToFinished-14.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-14.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAsuBf,CAAC;AACW,QAAA,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
+{"version":3,"file":"moveToFinished-14.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-14.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAuwBf,CAAC;AACW,QAAA,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}