bullmq 5.53.0 → 5.53.2

package/dist/cjs/commands/{moveStalledJobsToWait-9.lua → moveStalledJobsToWait-8.lua}

@@ -5,12 +5,11 @@
       KEYS[1] 'stalled' (SET)
       KEYS[2] 'wait',   (LIST)
       KEYS[3] 'active', (LIST)
-      KEYS[4] 'failed', (ZSET)
-      KEYS[5] 'stalled-check', (KEY)
-      KEYS[6] 'meta', (KEY)
-      KEYS[7] 'paused', (LIST)
-      KEYS[8] 'marker'
-      KEYS[9] 'event stream' (STREAM)
+      KEYS[4] 'stalled-check', (KEY)
+      KEYS[5] 'meta', (KEY)
+      KEYS[6] 'paused', (LIST)
+      KEYS[7] 'marker'
+      KEYS[8] 'event stream' (STREAM)
 
       ARGV[1]  Max stalled job count
       ARGV[2]  queue.toKey('')
@@ -26,27 +25,23 @@ local rcall = redis.call
 --- @include "includes/addJobInTargetList"
 --- @include "includes/batches"
 --- @include "includes/getTargetQueueList"
---- @include "includes/moveChildFromDependenciesIfNeeded"
---- @include "includes/removeDeduplicationKeyIfNeededOnFinalization"
---- @include "includes/removeJobsOnFail"
 --- @include "includes/trimEvents"
 
 local stalledKey = KEYS[1]
 local waitKey = KEYS[2]
 local activeKey = KEYS[3]
-local failedKey = KEYS[4]
-local stalledCheckKey = KEYS[5]
-local metaKey = KEYS[6]
-local pausedKey = KEYS[7]
-local markerKey = KEYS[8]
-local eventStreamKey = KEYS[9]
+local stalledCheckKey = KEYS[4]
+local metaKey = KEYS[5]
+local pausedKey = KEYS[6]
+local markerKey = KEYS[7]
+local eventStreamKey = KEYS[8]
 local maxStalledJobCount = tonumber(ARGV[1])
 local queueKeyPrefix = ARGV[2]
 local timestamp = ARGV[3]
 local maxCheckTime = ARGV[4]
 
 if rcall("EXISTS", stalledCheckKey) == 1 then
-    return {{}, {}}
+    return {}
 end
 
 rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
@@ -57,7 +52,6 @@ trimEvents(metaKey, eventStreamKey)
 -- Move all stalled jobs to wait
 local stalling = rcall('SMEMBERS', stalledKey)
 local stalled = {}
-local failed = {}
 if (#stalling > 0) then
     rcall('DEL', stalledKey)
 
@@ -78,36 +72,25 @@ if (#stalling > 0) then
                 if (removed > 0) then
                     -- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
                     local stalledCount = rcall("HINCRBY", jobKey, "stc", 1)
-                    if (stalledCount > maxStalledJobCount) then
-                        local jobAttributes = rcall("HMGET", jobKey, "opts", "parent", "deid")
+                    if stalledCount > maxStalledJobCount then
+                        local jobAttributes = rcall("HMGET", jobKey, "opts", "parent")
                         local rawOpts = jobAttributes[1]
                         local rawParentData = jobAttributes[2]
                         local opts = cjson.decode(rawOpts)
-                        rcall("ZADD", failedKey, timestamp, jobId)
-                        removeDeduplicationKeyIfNeededOnFinalization(queueKeyPrefix, jobAttributes[3], jobId)
 
                         local failedReason = "job stalled more than allowable limit"
-                        rcall("HMSET", jobKey, "failedReason", failedReason, "finishedOn", timestamp)
-                        rcall("XADD", eventStreamKey, "*", "event", "failed", "jobId", jobId, 'prev', 'active',
-                            'failedReason', failedReason)
-
-                        moveChildFromDependenciesIfNeeded(rawParentData, jobKey, failedReason, timestamp)
-
-                        removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestamp)
-
-                        table.insert(failed, jobId)
-                    else
-                        local target, isPausedOrMaxed = getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
+                        rcall("HSET", jobKey, "defa", failedReason)
+                    end
+                    local target, isPausedOrMaxed = getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
 
-                        -- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
-                        addJobInTargetList(target, markerKey, "RPUSH", isPausedOrMaxed, jobId)
+                    -- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
+                    addJobInTargetList(target, markerKey, "RPUSH", isPausedOrMaxed, jobId)
 
-                        rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId", jobId, 'prev', 'active')
+                    rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId", jobId, 'prev', 'active')
 
-                        -- Emit the stalled event
-                        rcall("XADD", eventStreamKey, "*", "event", "stalled", "jobId", jobId)
-                        table.insert(stalled, jobId)
-                    end
+                    -- Emit the stalled event
+                    rcall("XADD", eventStreamKey, "*", "event", "stalled", "jobId", jobId)
+                    table.insert(stalled, jobId)
                 end
             end
         end
@@ -123,4 +106,4 @@ if (#active > 0) then
     end
 end
 
-return {failed, stalled}
+return stalled

package/dist/cjs/scripts/addJobScheduler-11.js

@@ -394,6 +394,7 @@ local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMilli
     table.insert(optionalValues, "data")
     table.insert(optionalValues, templateData)
   end
+  rcall("DEL", schedulerKey) -- remove all attributes and then re-insert new ones
   rcall("HMSET", schedulerKey, "name", opts['name'], "ic", 1, unpack(optionalValues))
 end
 -- If we are overriding a repeatable job we must delete the delayed job for
@@ -405,7 +406,7 @@ local maxEvents = getOrSetMaxEvents(metaKey)
 local function removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey, jobId,
     metaKey, eventsKey)
     if rcall("ZSCORE", delayedKey, jobId) then
-        removeJob(nextDelayedJobId, true, prefixKey, true --[[remove debounce key]] )
+        removeJob(jobId, true, prefixKey, true --[[remove debounce key]] )
         rcall("ZREM", delayedKey, jobId)
         return true
     elseif rcall("ZSCORE", prioritizedKey, jobId) then

package/dist/cjs/scripts/addJobScheduler-11.js.map

@@ -1 +1 @@
-{"version":3,"file":"addJobScheduler-11.js","sourceRoot":"","sources":["../../../src/scripts/addJobScheduler-11.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkcf,CAAC;AACW,QAAA,eAAe,GAAG;IAC7B,IAAI,EAAE,iBAAiB;IACvB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
+{"version":3,"file":"addJobScheduler-11.js","sourceRoot":"","sources":["../../../src/scripts/addJobScheduler-11.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAmcf,CAAC;AACW,QAAA,eAAe,GAAG;IAC7B,IAAI,EAAE,iBAAiB;IACvB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}

package/dist/cjs/scripts/index.js

@@ -27,7 +27,7 @@ tslib_1.__exportStar(require("./isJobInList-1"), exports);
 tslib_1.__exportStar(require("./isMaxed-2"), exports);
 tslib_1.__exportStar(require("./moveJobFromActiveToWait-9"), exports);
 tslib_1.__exportStar(require("./moveJobsToWait-8"), exports);
-tslib_1.__exportStar(require("./moveStalledJobsToWait-9"), exports);
+tslib_1.__exportStar(require("./moveStalledJobsToWait-8"), exports);
 tslib_1.__exportStar(require("./moveToActive-11"), exports);
 tslib_1.__exportStar(require("./moveToDelayed-8"), exports);
 tslib_1.__exportStar(require("./moveToFinished-14"), exports);

package/dist/cjs/scripts/moveStalledJobsToWait-8.js

@@ -0,0 +1,171 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.moveStalledJobsToWait = void 0;
+const content = `--[[
+  Move stalled jobs to wait.
+    Input:
+      KEYS[1] 'stalled' (SET)
+      KEYS[2] 'wait',   (LIST)
+      KEYS[3] 'active', (LIST)
+      KEYS[4] 'stalled-check', (KEY)
+      KEYS[5] 'meta', (KEY)
+      KEYS[6] 'paused', (LIST)
+      KEYS[7] 'marker'
+      KEYS[8] 'event stream' (STREAM)
+      ARGV[1]  Max stalled job count
+      ARGV[2]  queue.toKey('')
+      ARGV[3]  timestamp
+      ARGV[4]  max check time
+    Events:
+      'stalled' with stalled job id.
+]]
+local rcall = redis.call
+-- Includes
+--[[
+  Function to add job in target list and add marker if needed.
+]]
+-- Includes
+--[[
+  Add marker if needed when a job is available.
+]]
+local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
+  if not isPausedOrMaxed then
+    rcall("ZADD", markerKey, 0, "0")
+  end  
+end
+local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
+  rcall(pushCmd, targetKey, jobId)
+  addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
+end
+--[[
+  Function to loop in batches.
+  Just a bit of warning, some commands as ZREM
+  could receive a maximum of 7000 parameters per call.
+]]
+local function batches(n, batchSize)
+  local i = 0
+  return function()
+    local from = i * batchSize + 1
+    i = i + 1
+    if (from <= n) then
+      local to = math.min(from + batchSize - 1, n)
+      return from, to
+    end
+  end
+end
+--[[
+  Function to check for the meta.paused key to decide if we are paused or not
+  (since an empty list and !EXISTS are not really the same).
+]]
+local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
+  local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
+  if queueAttributes[1] then
+    return pausedKey, true
+  else
+    if queueAttributes[2] then
+      local activeCount = rcall("LLEN", activeKey)
+      if activeCount >= tonumber(queueAttributes[2]) then
+        return waitKey, true
+      else
+        return waitKey, false
+      end
+    end
+  end
+  return waitKey, false
+end
+--[[
+  Function to trim events, default 10000.
+]]
+-- Includes
+--[[
+  Function to get max events value or set by default 10000.
+]]
+local function getOrSetMaxEvents(metaKey)
+  local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
+  if not maxEvents then
+    maxEvents = 10000
+    rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
+  end
+  return maxEvents
+end
+local function trimEvents(metaKey, eventStreamKey)
+  local maxEvents = getOrSetMaxEvents(metaKey)
+  if maxEvents then
+    rcall("XTRIM", eventStreamKey, "MAXLEN", "~", maxEvents)
+  else
+    rcall("XTRIM", eventStreamKey, "MAXLEN", "~", 10000)
+  end
+end
+local stalledKey = KEYS[1]
+local waitKey = KEYS[2]
+local activeKey = KEYS[3]
+local stalledCheckKey = KEYS[4]
+local metaKey = KEYS[5]
+local pausedKey = KEYS[6]
+local markerKey = KEYS[7]
+local eventStreamKey = KEYS[8]
+local maxStalledJobCount = tonumber(ARGV[1])
+local queueKeyPrefix = ARGV[2]
+local timestamp = ARGV[3]
+local maxCheckTime = ARGV[4]
+if rcall("EXISTS", stalledCheckKey) == 1 then
+    return {}
+end
+rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
+-- Trim events before emiting them to avoid trimming events emitted in this script
+trimEvents(metaKey, eventStreamKey)
+-- Move all stalled jobs to wait
+local stalling = rcall('SMEMBERS', stalledKey)
+local stalled = {}
+if (#stalling > 0) then
+    rcall('DEL', stalledKey)
+    -- Remove from active list
+    for i, jobId in ipairs(stalling) do
+        -- Markers in waitlist DEPRECATED in v5: Remove in v6.
+        if string.sub(jobId, 1, 2) == "0:" then
+            -- If the jobId is a delay marker ID we just remove it.
+            rcall("LREM", activeKey, 1, jobId)
+        else
+            local jobKey = queueKeyPrefix .. jobId
+            -- Check that the lock is also missing, then we can handle this job as really stalled.
+            if (rcall("EXISTS", jobKey .. ":lock") == 0) then
+                --  Remove from the active queue.
+                local removed = rcall("LREM", activeKey, 1, jobId)
+                if (removed > 0) then
+                    -- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
+                    local stalledCount = rcall("HINCRBY", jobKey, "stc", 1)
+                    if stalledCount > maxStalledJobCount then
+                        local jobAttributes = rcall("HMGET", jobKey, "opts", "parent")
+                        local rawOpts = jobAttributes[1]
+                        local rawParentData = jobAttributes[2]
+                        local opts = cjson.decode(rawOpts)
+                        local failedReason = "job stalled more than allowable limit"
+                        rcall("HSET", jobKey, "defa", failedReason)
+                    end
+                    local target, isPausedOrMaxed = getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
+                    -- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
+                    addJobInTargetList(target, markerKey, "RPUSH", isPausedOrMaxed, jobId)
+                    rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId", jobId, 'prev', 'active')
+                    -- Emit the stalled event
+                    rcall("XADD", eventStreamKey, "*", "event", "stalled", "jobId", jobId)
+                    table.insert(stalled, jobId)
+                end
+            end
+        end
+    end
+end
+-- Mark potentially stalled jobs
+local active = rcall('LRANGE', activeKey, 0, -1)
+if (#active > 0) then
+    for from, to in batches(#active, 7000) do
+        rcall('SADD', stalledKey, unpack(active, from, to))
+    end
+end
+return stalled
+`;
+exports.moveStalledJobsToWait = {
+    name: 'moveStalledJobsToWait',
+    content,
+    keys: 8,
+};
+//# sourceMappingURL=moveStalledJobsToWait-8.js.map

package/dist/cjs/scripts/moveStalledJobsToWait-8.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"moveStalledJobsToWait-8.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAiKf,CAAC;AACW,QAAA,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}