bullmq 2.3.0 → 2.3.2

package/dist/esm/commands/extendLock-2.lua

@@ -0,0 +1,23 @@
+--[[
+  Extend lock and removes the job from the stalled set.
+
+  Input:
+    KEYS[1] 'lock',
+    KEYS[2] 'stalled'
+
+    ARGV[1]  token
+    ARGV[2]  lock duration in milliseconds
+    ARGV[3]  jobid
+
+  Output:
+    "1" if lock extented succesfully.
+]]
+local rcall = redis.call
+if rcall("GET", KEYS[1]) == ARGV[1] then
+  --   if rcall("SET", KEYS[1], ARGV[1], "PX", ARGV[2], "XX") then
+  if rcall("SET", KEYS[1], ARGV[1], "PX", ARGV[2]) then
+    rcall("SREM", KEYS[2], ARGV[3])
+    return 1
+  end
+end
+return 0

package/dist/esm/commands/getState-7.lua

@@ -0,0 +1,57 @@
+--[[
+  Get a job state
+
+  Input: 
+    KEYS[1] 'completed' key,
+    KEYS[2] 'failed' key
+    KEYS[3] 'delayed' key
+    KEYS[4] 'active' key
+    KEYS[5] 'wait' key
+    KEYS[6] 'paused' key
+    KEYS[7] waitChildrenKey key
+
+    ARGV[1] job id
+  Output:
+    'completed'
+    'failed'
+    'delayed'
+    'active'
+    'waiting'
+    'waiting-children'
+    'unknown'
+]]
+if redis.call("ZSCORE", KEYS[1], ARGV[1]) ~= false then
+  return "completed"
+end
+
+if redis.call("ZSCORE", KEYS[2], ARGV[1]) ~= false then
+  return "failed"
+end
+
+if redis.call("ZSCORE", KEYS[3], ARGV[1]) ~= false then
+  return "delayed"
+end
+
+-- Includes
+--- @include "includes/checkItemInList"
+
+local active_items = redis.call("LRANGE", KEYS[4] , 0, -1)
+if checkItemInList(active_items, ARGV[1]) ~= nil then
+  return "active"
+end
+
+local wait_items = redis.call("LRANGE", KEYS[5] , 0, -1)
+if checkItemInList(wait_items, ARGV[1]) ~= nil then
+  return "waiting"
+end
+
+local paused_items = redis.call("LRANGE", KEYS[6] , 0, -1)
+if checkItemInList(paused_items, ARGV[1]) ~= nil then
+  return "waiting"
+end
+
+if redis.call("ZSCORE", KEYS[7], ARGV[1]) ~= false then
+  return "waiting-children"
+end
+
+return "unknown"

package/dist/esm/commands/getStateV2-7.lua

@@ -0,0 +1,51 @@
+--[[
+  Get a job state
+
+  Input: 
+    KEYS[1] 'completed' key,
+    KEYS[2] 'failed' key
+    KEYS[3] 'delayed' key
+    KEYS[4] 'active' key
+    KEYS[5] 'wait' key
+    KEYS[6] 'paused' key
+    KEYS[7] waitChildrenKey key
+
+    ARGV[1] job id
+  Output:
+    'completed'
+    'failed'
+    'delayed'
+    'active'
+    'waiting'
+    'waiting-children'
+    'unknown'
+]]
+if redis.call("ZSCORE", KEYS[1], ARGV[1]) ~= false then
+  return "completed"
+end
+
+if redis.call("ZSCORE", KEYS[2], ARGV[1]) ~= false then
+  return "failed"
+end
+
+if redis.call("ZSCORE", KEYS[3], ARGV[1]) ~= false then
+  return "delayed"
+end
+
+if redis.call("LPOS", KEYS[4] , ARGV[1]) ~= false then
+  return "active"
+end
+
+if redis.call("LPOS", KEYS[5] , ARGV[1]) ~= false then
+  return "waiting"
+end
+
+if redis.call("LPOS", KEYS[6] , ARGV[1]) ~= false then
+  return "waiting"
+end
+
+if redis.call("ZSCORE", KEYS[7] , ARGV[1]) ~= false then
+  return "waiting-children"
+end
+
+return "unknown"

package/dist/esm/commands/includes/addJobWithPriority.lua

@@ -0,0 +1,16 @@
+--[[
+  Function to add job considering priority.
+]]
+
+local function addJobWithPriority(priorityKey, priority, targetKey, jobId)
+  rcall("ZADD", priorityKey, priority, jobId)
+  local count = rcall("ZCOUNT", priorityKey, 0, priority)
+
+  local len = rcall("LLEN", targetKey)
+  local id = rcall("LINDEX", targetKey, len - (count - 1))
+  if id then
+    rcall("LINSERT", targetKey, "BEFORE", id, jobId)
+  else
+    rcall("RPUSH", targetKey, jobId)
+  end
+end

package/dist/esm/commands/includes/batches.lua

@@ -0,0 +1,18 @@
+--[[
+  Function to loop in batches.
+  Just a bit of warning, some commands as ZREM
+  could receive a maximum of 7000 parameters per call.
+]]
+
+local function batches(n, batchSize)
+  local i = 0
+
+  return function()
+    local from = i * batchSize + 1
+    i = i + 1
+    if (from <= n) then
+      local to = math.min(from + batchSize - 1, n)
+      return from, to
+    end
+  end
+end

package/dist/esm/commands/includes/checkItemInList.lua

@@ -0,0 +1,12 @@
+--[[
+  Functions to check if a item belongs to a list.
+]]
+
+local function checkItemInList(list, item)
+  for _, v in pairs(list) do
+    if v == item then
+      return 1
+    end
+  end
+  return nil
+end

package/dist/esm/commands/includes/checkStalledJobs.lua

@@ -0,0 +1,137 @@
+--[[
+  Move stalled jobs to wait.
+
+    Input:
+      stalledKey 'stalled' (SET)
+      waitKey 'wait',   (LIST)
+      activeKey 'active', (LIST)
+      failedKey 'failed', (ZSET)
+      stalledCheckKey 'stalled-check', (KEY)
+      metaKey 'meta', (KEY)
+      pausedKey 'paused', (LIST)
+      eventStreamKey 'event stream' (STREAM)
+
+      maxStalledJobCount  Max stalled job count
+      queueKeyPrefix  queue.toKey('')
+      timestamp  timestamp
+      maxCheckTime  max check time
+
+    Events:
+      'stalled' with stalled job id.
+]] local rcall = redis.call
+
+-- Includes
+--- @include "batches"
+--- @include "getTargetQueueList"
+--- @include "removeJob"
+--- @include "removeJobsByMaxAge"
+--- @include "removeJobsByMaxCount"
+--- @include "trimEvents"
+
+-- Check if we need to check for stalled jobs now.
+
+local function checkStalledJobs(stalledKey, waitKey, activeKey, failedKey, stalledCheckKey,
+    metaKey, pausedKey, eventStreamKey, maxStalledJobCount, queueKeyPrefix, timestamp, maxCheckTime)
+    if rcall("EXISTS", stalledCheckKey) == 1 then 
+        return {{}, {}}
+    end
+
+    rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
+
+    -- Trim events before emiting them to avoid trimming events emitted in this script
+    trimEvents(metaKey, eventStreamKey)
+
+    -- Move all stalled jobs to wait
+    local stalling = rcall('SMEMBERS', stalledKey)
+    local stalled = {}
+    local failed = {}
+    if (#stalling > 0) then
+        rcall('DEL', stalledKey)
+
+        local MAX_STALLED_JOB_COUNT = tonumber(maxStalledJobCount)
+
+        -- Remove from active list
+        for i, jobId in ipairs(stalling) do
+
+            if jobId == '0' then
+                -- If the jobId is a delay marker ID we just remove it.
+                local removed = rcall("LREM", activeKey, 1, jobId)
+            else
+                local jobKey = queueKeyPrefix .. jobId
+
+                -- Check that the lock is also missing, then we can handle this job as really stalled.
+                if (rcall("EXISTS", jobKey .. ":lock") == 0) then
+                    --  Remove from the active queue.
+                    local removed = rcall("LREM", activeKey, 1, jobId)
+
+                    if (removed > 0) then
+                        -- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
+                        local stalledCount =
+                            rcall("HINCRBY", jobKey, "stalledCounter", 1)
+                        if (stalledCount > MAX_STALLED_JOB_COUNT) then
+                            local rawOpts = rcall("HGET", jobKey, "opts")
+                            local opts = cjson.decode(rawOpts)
+                            local removeOnFailType = type(opts["removeOnFail"])
+                            rcall("ZADD", failedKey, timestamp, jobId)
+                            local failedReason =
+                                "job stalled more than allowable limit"
+                            rcall("HMSET", jobKey, "failedReason", failedReason,
+                                "finishedOn", timestamp)
+                            rcall("XADD", eventStreamKey, "*", "event", "failed", "jobId",
+                                jobId, 'prev', 'active', 'failedReason',
+                                failedReason)
+
+                            if removeOnFailType == "number" then
+                                removeJobsByMaxCount(opts["removeOnFail"], failedKey,
+                                                    queueKeyPrefix)
+                            elseif removeOnFailType == "boolean" then
+                                if opts["removeOnFail"] then
+                                    removeJob(jobId, false, queueKeyPrefix)
+                                    rcall("ZREM", failedKey, jobId)
+                                end
+                            elseif removeOnFailType ~= "nil" then
+                                local maxAge = opts["removeOnFail"]["age"]
+                                local maxCount = opts["removeOnFail"]["count"]
+
+                                if maxAge ~= nil then
+                                    removeJobsByMaxAge(timestamp, maxAge, failedKey,
+                                                    queueKeyPrefix)
+                                end
+
+                                if maxCount ~= nil and maxCount > 0 then
+                                    removeJobsByMaxCount(maxCount, failedKey, queueKeyPrefix)
+                                end
+                            end
+
+                            table.insert(failed, jobId)
+                        else
+                            local target = getTargetQueueList(metaKey, waitKey,
+                                                            pausedKey)
+
+                            -- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
+                            rcall("RPUSH", target, jobId)
+                            rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
+                                jobId, 'prev', 'active')
+
+                            -- Emit the stalled event
+                            rcall("XADD", eventStreamKey, "*", "event", "stalled", "jobId",
+                                jobId)
+                            table.insert(stalled, jobId)
+                        end
+                    end
+                end
+            end
+        end
+    end
+
+    -- Mark potentially stalled jobs
+    local active = rcall('LRANGE', activeKey, 0, -1)
+
+    if (#active > 0) then
+        for from, to in batches(#active, 7000) do
+            rcall('SADD', stalledKey, unpack(active, from, to))
+        end
+    end
+
+    return {failed, stalled}
+end

package/dist/esm/commands/includes/cleanList.lua

@@ -0,0 +1,50 @@
+--[[
+  Function to clean job list.
+  Returns jobIds and deleted count number.
+]]
+
+-- Includes
+--- @include "getTimestamp"
+--- @include "removeJob"
+
+local function cleanList(listKey, jobKeyPrefix, rangeStart, rangeEnd,
+  timestamp, isWaiting)
+  local jobs = rcall("LRANGE", listKey, rangeStart, rangeEnd)
+  local deleted = {}
+  local deletedCount = 0
+  local jobTS
+  local deletionMarker = ''
+  local jobIdsLen = #jobs
+  for i, job in ipairs(jobs) do
+    if limit > 0 and deletedCount >= limit then
+      break
+    end
+  
+    local jobKey = jobKeyPrefix .. job
+    if (isWaiting or rcall("EXISTS", jobKey .. ":lock") == 0) then
+      -- Find the right timestamp of the job to compare to maxTimestamp:
+      -- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
+      -- * processedOn represents when the job was last attempted, but it doesn't get populated until
+      --   the job is first tried
+      -- * timestamp is the original job submission time
+      -- Fetch all three of these (in that order) and use the first one that is set so that we'll leave jobs
+      -- that have been active within the grace period:
+      jobTS = getTimestamp(jobKey, {"finishedOn", "processedOn", "timestamp"})
+      if (not jobTS or jobTS < timestamp) then
+        -- replace the entry with a deletion marker; the actual deletion will
+        -- occur at the end of the script
+        rcall("LSET", listKey, rangeEnd - jobIdsLen + i, deletionMarker)
+        removeJob(job, true, jobKeyPrefix)
+        if isWaiting then
+          rcall("ZREM", jobKeyPrefix .. "priority", job)
+        end
+        deletedCount = deletedCount + 1
+        table.insert(deleted, job)
+      end
+    end
+  end
+
+  rcall("LREM", listKey, 0, deletionMarker)
+
+  return {deleted, deletedCount}
+end

package/dist/esm/commands/includes/cleanSet.lua

@@ -0,0 +1,50 @@
+--[[
+  Function to clean job set.
+  Returns jobIds and deleted count number.
+]]
+
+-- Includes
+--- @include "batches"
+--- @include "getTimestamp"
+--- @include "removeJob"
+
+-- We use ZRANGEBYSCORE to make the case where we're deleting a limited number
+-- of items in a sorted set only run a single iteration. If we simply used
+-- ZRANGE, we may take a long time traversing through jobs that are within the
+-- grace period.
+local function getJobs(setKey, rangeStart, rangeEnd, maxTimestamp, limit)
+  if limit > 0 then
+    return rcall("ZRANGEBYSCORE", setKey, 0, maxTimestamp, "LIMIT", 0, limit)
+  else
+    return rcall("ZRANGE", setKey, rangeStart, rangeEnd)
+  end
+end
+
+local function cleanSet(setKey, jobKeyPrefix, rangeStart, rangeEnd, timestamp, limit, attributes)
+  local jobs = getJobs(setKey, rangeStart, rangeEnd, timestamp, limit)
+  local deleted = {}
+  local deletedCount = 0
+  local jobTS
+  for i, job in ipairs(jobs) do
+    if limit > 0 and deletedCount >= limit then
+      break
+    end
+  
+    local jobKey = jobKeyPrefix .. job
+    -- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
+    jobTS = getTimestamp(jobKey, attributes)
+    if (not jobTS or jobTS < timestamp) then
+      removeJob(job, true, jobKeyPrefix)
+      deletedCount = deletedCount + 1
+      table.insert(deleted, job)
+    end
+  end
+
+  if(#deleted > 0) then
+    for from, to in batches(#deleted, 7000) do
+      rcall("ZREM", setKey, unpack(deleted, from, to))
+    end
+  end
+  
+  return {deleted, deletedCount}
+end

package/dist/esm/commands/includes/collectMetrics.lua

@@ -0,0 +1,46 @@
+--[[
+  Functions to collect metrics based on a current and previous count of jobs.
+  Granualarity is fixed at 1 minute.
+]] 
+--- @include "batches"
+local function collectMetrics(metaKey, dataPointsList, maxDataPoints,
+                                 timestamp)
+    -- Increment current count
+    local count = rcall("HINCRBY", metaKey, "count", 1) - 1
+
+    -- Compute how many data points we need to add to the list, N.
+    local prevTS = rcall("HGET", metaKey, "prevTS")
+
+    if not prevTS then
+        -- If prevTS is nil, set it to the current timestamp
+        rcall("HSET", metaKey, "prevTS", timestamp, "prevCount", 0)
+        return
+    end
+
+    local N = math.floor((timestamp - prevTS) / 60000)
+
+    if N > 0 then
+        local delta = count - rcall("HGET", metaKey, "prevCount")
+        -- If N > 1, add N-1 zeros to the list
+        if N > 1 then
+            local points = {}
+            points[1] = delta
+            for i = 2, N do
+                points[i] = 0
+            end
+
+            for from, to in batches(#points, 7000) do
+                rcall("LPUSH", dataPointsList, unpack(points, from, to))
+            end
+        else
+            -- LPUSH delta to the list
+            rcall("LPUSH", dataPointsList, delta)
+        end
+
+        -- LTRIM to keep list to its max size
+        rcall("LTRIM", dataPointsList, 0, maxDataPoints - 1)
+
+        -- update prev count with current count
+        rcall("HSET", metaKey, "prevCount", count, "prevTS", timestamp)
+    end
+end

package/dist/esm/commands/includes/destructureJobKey.lua

@@ -0,0 +1,12 @@
+--[[
+  Functions to destructure job key.
+  Just a bit of warning, these functions may be a bit slow and affect performance significantly.
+]]
+
+local getJobIdFromKey = function (jobKey)
+  return string.match(jobKey, ".*:(.*)")
+end
+
+local getJobKeyPrefix = function (jobKey, jobId)
+  return string.sub(jobKey, 0, #jobKey - #jobId)
+end

package/dist/esm/commands/includes/getNextDelayedTimestamp.lua

@@ -0,0 +1,13 @@
+--[[
+  Function to return the next delayed job timestamp.
+]] 
+local function getNextDelayedTimestamp(delayedKey)
+    local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
+    if #result then
+      local nextTimestamp = tonumber(result[2])
+      if (nextTimestamp ~= nil) then 
+        nextTimestamp = nextTimestamp / 0x1000
+      end
+      return nextTimestamp
+    end
+end

package/dist/esm/commands/includes/getTargetQueueList.lua

@@ -0,0 +1,12 @@
+--[[
+  Function to check for the meta.paused key to decide if we are paused or not
+  (since an empty list and !EXISTS are not really the same).
+]]
+
+local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
+  if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
+    return waitKey
+  else
+    return pausedKey
+  end
+end

package/dist/esm/commands/includes/getTimestamp.lua

@@ -0,0 +1,19 @@
+--[[
+  Function to get the latest saved timestamp.
+]]
+
+local function getTimestamp(jobKey, attributes)
+  if #attributes == 1 then
+    return rcall("HGET", jobKey, attributes[1])
+  end
+
+  local jobTs
+  for _, ts in ipairs(rcall("HMGET", jobKey, unpack(attributes))) do
+    if (ts) then
+      jobTs = ts
+      break
+    end
+  end
+  
+  return jobTs
+end

package/dist/esm/commands/includes/getZSetItems.lua

@@ -0,0 +1,7 @@
+--[[
+  Function to get ZSet items.
+]]
+
+local function getZSetItems(keyName, max)
+  return rcall('ZRANGE', keyName, 0, max - 1)
+end

package/dist/esm/commands/includes/isLocked.lua

@@ -0,0 +1,31 @@
+--[[
+  Function to recursively check if there are no locks
+  on the jobs to be removed.
+
+  returns:
+    boolean
+]]
+
+local function isLocked( prefix, jobId)
+  local jobKey = prefix .. jobId;
+
+  -- Check if this job is locked
+  local lockKey = jobKey .. ':lock'
+  local lock = rcall("GET", lockKey)
+  if not lock then
+      local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
+      if (#dependencies > 0) then
+          for i, childJobKey in ipairs(dependencies) do
+              -- We need to get the jobId for this job.
+              local childJobId = getJobIdFromKey(childJobKey)
+              local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
+              local result = isLocked( childJobPrefix, childJobId )
+              if result then
+                  return true
+              end
+          end
+      end
+      return false
+  end
+  return true
+end

package/dist/esm/commands/includes/moveJobFromWaitToActive.lua

@@ -0,0 +1,84 @@
+
+--[[
+  Function to move job from wait state to active.
+  Input:
+    keys[1] wait key
+    keys[2] active key
+    keys[3] priority key
+    keys[4] stream events key
+    keys[5] stalled key
+
+    -- Rate limiting
+    keys[6] rate limiter key
+    keys[7] delayed key
+
+    opts - token - lock token
+    opts - lockDuration
+    opts - limiter
+]]
+
+local function moveJobFromWaitToActive(keys, keyPrefix, jobId, processedOn, opts)
+  -- Check if we need to perform rate limiting.
+  local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
+
+  if(maxJobs) then
+    local rateLimiterKey = keys[6];
+
+    local groupKey
+    local groupKeyOpt = opts['limiter'] and opts['limiter']['groupKey'] or ""
+    if groupKeyOpt ~= "" then
+      groupKey = string.match(jobId, "[^:]+$")
+      if groupKey ~= jobId then
+        rateLimiterKey = rateLimiterKey .. ":" .. groupKey
+      end
+    end
+
+    local jobCounter
+    
+    if groupKey ~= nil then
+      if rateLimiterKey ~= keys[6] then
+        jobCounter = tonumber(rcall("INCR", rateLimiterKey))
+      end
+    else
+      jobCounter = tonumber(rcall("INCR", rateLimiterKey))
+    end
+
+    local limiterDuration = opts['limiter'] and opts['limiter']['duration']
+    -- check if rate limit hit
+    if jobCounter ~= nil and jobCounter > maxJobs then
+      local exceedingJobs = jobCounter - maxJobs
+      local expireTime = tonumber(rcall("PTTL", rateLimiterKey))
+      local delay = expireTime + ((exceedingJobs - 1) * limiterDuration) / maxJobs;
+      local timestamp = delay + tonumber(processedOn)
+
+      -- put job into delayed queue
+      rcall("ZADD", keys[7], timestamp * 0x1000 + bit.band(jobCounter, 0xfff), jobId);
+      rcall("XADD", keys[4], "*", "event", "delayed", "jobId", jobId, "delay", timestamp);
+
+      -- remove from active queue
+      rcall("LREM", keys[2], 1, jobId)
+
+      -- Return when we can process more jobs
+      return expireTime
+    else
+      if jobCounter == 1 then
+        rcall("PEXPIRE", rateLimiterKey, limiterDuration)
+      end
+    end
+  end
+
+  local jobKey = keyPrefix .. jobId
+  local lockKey = jobKey .. ':lock'
+
+  -- get a lock
+  if opts['token'] ~= "0" then
+    rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
+  end
+
+  rcall("ZREM", keys[3], jobId) -- remove from priority
+  rcall("XADD", keys[4], "*", "event", "active", "jobId", jobId, "prev", "waiting")
+  rcall("HSET", jobKey, "processedOn", processedOn)
+  rcall("HINCRBY", jobKey, "attemptsMade", 1)
+
+  return {rcall("HGETALL", jobKey), jobId} -- get job data
+end

package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua

@@ -0,0 +1,28 @@
+--[[
+  Function to recursively move from waitingChildren to failed.
+]]
+
+local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
+  if rcall("ZREM", parentQueueKey .. ":waiting-children", parentId) == 1 then
+    rcall("ZADD", parentQueueKey .. ":failed", timestamp, parentId)
+    local failedReason = "child " .. jobIdKey .. " failed"
+    rcall("HMSET", parentKey, "failedReason", failedReason, "finishedOn", timestamp)
+    rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason",
+      failedReason, "prev", "waiting-children")
+
+    local rawParentData = rcall("HGET", parentKey, "parent")
+
+    if rawParentData ~= false then
+      local parentData = cjson.decode(rawParentData)
+      if parentData['fpof'] then
+        moveParentFromWaitingChildrenToFailed(
+          parentData['queueKey'],
+          parentData['queueKey'] .. ':' .. parentData['id'],
+          parentData['id'],
+          parentKey,
+          timestamp
+        )
+      end
+    end
+  end
+end