bullmq 5.41.5 → 5.41.7

package/dist/cjs/commands/addJobScheduler-10.lua

@@ -0,0 +1,130 @@
+--[[
+  Adds a job scheduler, i.e. a job factory that creates jobs based on a given schedule (repeat options).
+
+    Input:
+      KEYS[1]  'repeat' key
+      KEYS[2]  'delayed' key
+      KEYS[3]  'wait' key
+      KEYS[4]  'paused' key
+      KEYS[5]  'meta' key
+      KEYS[6]  'prioritized' key
+      KEYS[7]  'marker' key
+      KEYS[8]  'id' key
+      KEYS[9]  'events' key
+      KEYS[10] 'pc' priority counter
+      
+      ARGV[1] next milliseconds
+      ARGV[2] msgpacked options
+            [1]  name
+            [2]  tz?
+            [3]  patten?
+            [4]  endDate?
+            [5]  every?
+      ARGV[3] jobs scheduler id
+      ARGV[4] Json stringified template data
+      ARGV[5] mspacked template opts
+      ARGV[6] msgpacked delayed opts
+      ARGV[7] timestamp
+      ARGV[8] prefix key
+      ARGV[9] producer key
+
+      Output:
+        repeatableKey  - OK
+]]
+local rcall = redis.call
+local repeatKey = KEYS[1]
+local delayedKey = KEYS[2]
+local prioritizedKey = KEYS[6]
+
+local nextMillis = ARGV[1]
+local jobSchedulerId = ARGV[3]
+local templateOpts = cmsgpack.unpack(ARGV[5])
+local prefixKey = ARGV[8]
+
+-- Includes
+--- @include "includes/addDelayedJob"
+--- @include "includes/addJobWithPriority"
+--- @include "includes/getOrSetMaxEvents"
+--- @include "includes/getTargetQueueList"
+--- @include "includes/isQueuePaused"
+--- @include "includes/removeJob"
+--- @include "includes/storeJob"
+--- @include "includes/storeJobScheduler"
+
+-- If we are overriding a repeatable job we must delete the delayed job for
+-- the next iteration.
+local schedulerKey = repeatKey .. ":" .. jobSchedulerId
+local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
+local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
+local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
+
+if prevMillis ~= false then
+    local currentJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
+
+    if rcall("EXISTS", nextDelayedJobKey) ~= 1 or currentJobId == nextDelayedJobId then
+        if rcall("ZSCORE", delayedKey, currentJobId) ~= false then
+            removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
+            rcall("ZREM", delayedKey, currentJobId)
+        elseif rcall("ZSCORE", prioritizedKey, currentJobId) ~= false then
+            removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
+            rcall("ZREM", prioritizedKey, currentJobId)
+        else
+            if isQueuePaused(KEYS[5]) then
+                if rcall("LREM", KEYS[4], 1, currentJobId) > 0 then
+                    removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
+                end
+            else
+                if rcall("LREM", KEYS[3], 1, currentJobId) > 0 then
+                    removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
+                end
+            end
+        end
+    end
+end
+
+local schedulerOpts = cmsgpack.unpack(ARGV[2])
+storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, ARGV[4], templateOpts)
+
+if rcall("EXISTS", nextDelayedJobKey) ~= 1 then
+    local eventsKey = KEYS[9]
+    local metaKey = KEYS[5]
+    local maxEvents = getOrSetMaxEvents(metaKey)
+
+    rcall("INCR", KEYS[8])
+
+    local delayedOpts = cmsgpack.unpack(ARGV[6])
+
+    local delay, priority = storeJob(eventsKey, nextDelayedJobKey, nextDelayedJobId, schedulerOpts['name'], ARGV[4],
+        delayedOpts, ARGV[7], nil, nil, jobSchedulerId)
+
+    if delay ~= 0 then
+        addDelayedJob(nextDelayedJobId, delayedKey, eventsKey,
+            ARGV[7], maxEvents, KEYS[7], delay)
+    else
+        local isPaused = isQueuePaused(KEYS[5])
+      
+        -- Standard or priority add
+        if priority == 0 then
+            if isPaused then
+                -- LIFO or FIFO
+                local pushCmd = delayedOpts['lifo'] and 'RPUSH' or 'LPUSH'
+                rcall(pushCmd, KEYS[4], nextDelayedJobId)
+            else
+                -- LIFO or FIFO
+                local pushCmd = delayedOpts['lifo'] and 'RPUSH' or 'LPUSH'
+                rcall(pushCmd, KEYS[3], nextDelayedJobId)
+            end
+        else
+            -- Priority add
+            addJobWithPriority(KEYS[7], KEYS[6], priority, nextDelayedJobId, KEYS[10], isPaused)
+        end
+        -- Emit waiting event
+        rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents,  "*", "event", "waiting", "jobId", nextDelayedJobId)
+    end
+
+    if ARGV[9] ~= "" then
+        rcall("HSET", ARGV[9], "nrjid", nextDelayedJobId)
+    end
+
+    return nextDelayedJobId .. "" -- convert to string
+end

package/dist/cjs/commands/cleanJobsInSet-3.lua

@@ -4,7 +4,7 @@
   Input:
     KEYS[1]  set key,
     KEYS[2]  events stream key
-    KEYS[3]  job schedulers key
+    KEYS[3]  repeat key
 
     ARGV[1]  jobKey prefix
     ARGV[2]  timestamp
@@ -12,6 +12,7 @@
     ARGV[4]  set name, can be any of 'wait', 'active', 'paused', 'delayed', 'completed', or 'failed'
 ]]
 local rcall = redis.call
+local repeatKey = KEYS[3]
 local rangeStart = 0
 local rangeEnd = -1
 
@@ -34,20 +35,21 @@ end
 local result
 if ARGV[4] == "active" then
   result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], false --[[ hasFinished ]],
-                      KEYS[3])
+                      repeatKey)
 elseif ARGV[4] == "delayed" then
   rangeEnd = "+inf"
   result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
-                    {"processedOn", "timestamp"}, false  --[[ hasFinished ]], KEYS[3])
+                    {"processedOn", "timestamp"}, false  --[[ hasFinished ]], repeatKey)
 elseif ARGV[4] == "prioritized" then
   rangeEnd = "+inf"
   result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
-                    {"timestamp"}, false  --[[ hasFinished ]])
+                    {"timestamp"}, false  --[[ hasFinished ]], repeatKey)
 elseif ARGV[4] == "wait" or ARGV[4] == "paused" then
   result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], true --[[ hasFinished ]],
-                      KEYS[3])
+                      repeatKey)
 else
   rangeEnd = ARGV[2]
+  -- No need to pass repeat key as in that moment job won't be related to a job scheduler
   result = cleanSet(KEYS[1], ARGV[1], rangeEnd, ARGV[2], limit,
                     {"finishedOn"}, true  --[[ hasFinished ]])
 end

package/dist/cjs/commands/includes/addDelayedJob.lua

@@ -8,13 +8,9 @@
 -- Includes
 --- @include "addDelayMarkerIfNeeded"
 --- @include "getDelayedScore"
---- @include "storeJob"
 
-local function addDelayedJob(jobIdKey, jobId, delayedKey, eventsKey, name, data, opts, timestamp, repeatJobKey,
-  maxEvents, markerKey, parentKey, parentData)
-  -- Store the job.
-  local delay, priority = storeJob(eventsKey, jobIdKey, jobId, name, data,
-    opts, timestamp, parentKey, parentData, repeatJobKey)
+local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
+  maxEvents, markerKey, delay)
 
   local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
 
@@ -25,4 +21,3 @@ local function addDelayedJob(jobIdKey, jobId, delayedKey, eventsKey, name, data,
   -- mark that a delayed job is available
   addDelayMarkerIfNeeded(markerKey, delayedKey)
 end
-  

package/dist/cjs/commands/includes/cleanSet.lua

@@ -31,7 +31,7 @@ local function cleanSet(
         local jobKey = jobKeyPrefix .. job
         -- Extract a Job Scheduler Id from jobId ("repeat:job-scheduler-id:millis") 
         -- and check if it is in the scheduled jobs
-        if not isJobSchedulerJob(job, jobKey, jobSchedulersKey) then
+        if not (jobSchedulersKey and isJobSchedulerJob(job, jobKey, jobSchedulersKey)) then
             if isFinished then
                 removeJob(job, true, jobKeyPrefix, true --[[remove debounce key]] )
                 deletedCount = deletedCount + 1

package/dist/cjs/commands/includes/getOrSetMaxEvents.lua

@@ -2,10 +2,10 @@
   Function to get max events value or set by default 10000.
 ]]
 local function getOrSetMaxEvents(metaKey)
-    local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
-    if not maxEvents then
-        maxEvents = 10000
-        rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
-    end
-    return maxEvents
+  local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
+  if not maxEvents then
+    maxEvents = 10000
+    rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
+  end
+  return maxEvents
 end

package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua

@@ -7,8 +7,11 @@
 --- @include "removeDeduplicationKeyIfNeeded"
 --- @include "removeJobsOnFail"
 
-local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
-  if rcall("ZREM", parentQueueKey .. ":waiting-children", parentId) == 1 then
+local function moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
+  local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
+  local parentDelayedKey = parentQueueKey .. ":delayed"
+  if rcall("ZSCORE", parentWaitingChildrenKey, parentId) ~= false then
+    rcall("ZREM", parentWaitingChildrenKey, parentId)
     local parentQueuePrefix = parentQueueKey .. ":"
     local parentFailedKey = parentQueueKey .. ":failed"
     rcall("ZADD", parentFailedKey, timestamp, parentId)
@@ -48,6 +51,48 @@ local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey,
     local parentRawOpts = jobAttributes[3]
     local parentOpts = cjson.decode(parentRawOpts)
     
+    removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
+  elseif rcall("ZSCORE", parentDelayedKey, parentId) ~= false then
+    rcall("ZREM", parentDelayedKey, parentId)
+    local parentQueuePrefix = parentQueueKey .. ":"
+    local parentFailedKey = parentQueueKey .. ":failed"
+    rcall("ZADD", parentFailedKey, timestamp, parentId)
+    local failedReason = "child " .. jobIdKey .. " failed"
+    rcall("HMSET", parentKey, "failedReason", failedReason, "finishedOn", timestamp)
+    rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason",
+      failedReason, "prev", "delayed")
+
+    local jobAttributes = rcall("HMGET", parentKey, "parent", "deid", "opts")
+
+    removeDeduplicationKeyIfNeeded(parentQueueKey .. ":", jobAttributes[2])
+
+    if jobAttributes[1] then
+      local parentData = cjson.decode(jobAttributes[1])
+      if parentData['fpof'] then
+        moveParentFromWaitingChildrenToFailed(
+          parentData['queueKey'],
+          parentData['queueKey'] .. ':' .. parentData['id'],
+          parentData['id'],
+          parentKey,
+          timestamp
+        )
+      elseif parentData['idof'] or parentData['rdof'] then
+        local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
+        local grandParentDependenciesSet = grandParentKey .. ":dependencies"
+        if rcall("SREM", grandParentDependenciesSet, parentKey) == 1 then
+          moveParentToWaitIfNeeded(parentData['queueKey'], grandParentDependenciesSet,
+            grandParentKey, parentData['id'], timestamp)
+          if parentData['idof'] then
+            local grandParentFailedSet = grandParentKey .. ":failed"
+            rcall("HSET", grandParentFailedSet, parentKey, failedReason)
+          end
+        end
+      end
+    end
+
+    local parentRawOpts = jobAttributes[3]
+    local parentOpts = cjson.decode(parentRawOpts)
+    
     removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
   end
 end

package/dist/cjs/commands/moveStalledJobsToWait-9.lua

@@ -95,7 +95,7 @@ if (#stalling > 0) then
                               "failed", "jobId", jobId, 'prev', 'active',
                               'failedReason', failedReason)
 
-                        if rawParentData ~= false then
+                        if rawParentData then
                             if opts['fpof'] then
                                 local parentData = cjson.decode(rawParentData)
                                 moveParentFromWaitingChildrenToFailed(

package/dist/cjs/commands/moveToFinished-14.lua

@@ -103,7 +103,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
     local parentKey = jobAttributes[1] or ""
     local parentId = ""
     local parentQueueKey = ""
-    if jobAttributes[2] ~= false then
+    if jobAttributes[2] then
         local jsonDecodedParent = cjson.decode(jobAttributes[2])
         parentId = jsonDecodedParent['id']
         parentQueueKey = jsonDecodedParent['queueKey']

package/dist/cjs/scripts/addDelayedJob-6.js

@@ -103,41 +103,8 @@ local function getDelayedScore(delayedKey, timestamp, delay)
   end
   return minScore, delayedTimestamp
 end
---[[
-  Function to store a job
-]]
-local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
-                        parentKey, parentData, repeatJobKey)
-    local jsonOpts = cjson.encode(opts)
-    local delay = opts['delay'] or 0
-    local priority = opts['priority'] or 0
-    local debounceId = opts['de'] and opts['de']['id']
-    local optionalValues = {}
-    if parentKey ~= nil then
-        table.insert(optionalValues, "parentKey")
-        table.insert(optionalValues, parentKey)
-        table.insert(optionalValues, "parent")
-        table.insert(optionalValues, parentData)
-    end
-    if repeatJobKey ~= nil then
-        table.insert(optionalValues, "rjk")
-        table.insert(optionalValues, repeatJobKey)
-    end
-    if debounceId then
-        table.insert(optionalValues, "deid")
-        table.insert(optionalValues, debounceId)
-    end
-    rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
-          "timestamp", timestamp, "delay", delay, "priority", priority,
-          unpack(optionalValues))
-    rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
-    return delay, priority
-end
-local function addDelayedJob(jobIdKey, jobId, delayedKey, eventsKey, name, data, opts, timestamp, repeatJobKey,
-  maxEvents, markerKey, parentKey, parentData)
-  -- Store the job.
-  local delay, priority = storeJob(eventsKey, jobIdKey, jobId, name, data,
-    opts, timestamp, parentKey, parentData, repeatJobKey)
+local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
+  maxEvents, markerKey, delay)
   local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
   rcall("ZADD", delayedKey, score, jobId)
   rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
@@ -172,12 +139,12 @@ end
   Function to get max events value or set by default 10000.
 ]]
 local function getOrSetMaxEvents(metaKey)
-    local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
-    if not maxEvents then
-        maxEvents = 10000
-        rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
-    end
-    return maxEvents
+  local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
+  if not maxEvents then
+    maxEvents = 10000
+    rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
+  end
+  return maxEvents
 end
 --[[
   Function to handle the case when job is duplicated.
@@ -347,6 +314,36 @@ local function handleDuplicatedJob(jobKey, jobId, currentParentKey, currentParen
     "duplicated", "jobId", jobId)
   return jobId .. "" -- convert to string
 end
+--[[
+  Function to store a job
+]]
+local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
+                        parentKey, parentData, repeatJobKey)
+    local jsonOpts = cjson.encode(opts)
+    local delay = opts['delay'] or 0
+    local priority = opts['priority'] or 0
+    local debounceId = opts['de'] and opts['de']['id']
+    local optionalValues = {}
+    if parentKey ~= nil then
+        table.insert(optionalValues, "parentKey")
+        table.insert(optionalValues, parentKey)
+        table.insert(optionalValues, "parent")
+        table.insert(optionalValues, parentData)
+    end
+    if repeatJobKey ~= nil then
+        table.insert(optionalValues, "rjk")
+        table.insert(optionalValues, repeatJobKey)
+    end
+    if debounceId then
+        table.insert(optionalValues, "deid")
+        table.insert(optionalValues, debounceId)
+    end
+    rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
+          "timestamp", timestamp, "delay", delay, "priority", priority,
+          unpack(optionalValues))
+    rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
+    return delay, priority
+end
 if parentKey ~= nil then
     if rcall("EXISTS", parentKey) ~= 1 then return -5 end
     parentData = cjson.encode(parent)
@@ -373,8 +370,9 @@ local deduplicationJobId = deduplicateJob(opts['de'], jobId, deduplicationKey,
 if deduplicationJobId then
   return deduplicationJobId
 end
-addDelayedJob(jobIdKey, jobId, delayedKey, eventsKey, args[3], ARGV[2], opts, timestamp, repeatJobKey,
-  maxEvents, KEYS[1], parentKey, parentData)
+local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
+    opts, timestamp, parentKey, parentData, repeatJobKey)
+addDelayedJob(jobId, delayedKey, eventsKey, timestamp, maxEvents, KEYS[1], delay)
 -- Check if this job is a child of another job, if so add it to the parents dependencies
 if parentDependenciesKey ~= nil then
     rcall("SADD", parentDependenciesKey, jobIdKey)

package/dist/cjs/scripts/addDelayedJob-6.js.map

@@ -1 +1 @@
-{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2Xf,CAAC;AACW,QAAA,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
+{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyXf,CAAC;AACW,QAAA,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}