bullmq 1.59.0 → 1.59.4

package/dist/esm/commands/addJob-9.lua

@@ -0,0 +1,179 @@
+--[[
+  Adds a job to the queue by doing the following:
+    - Increases the job counter if needed.
+    - Creates a new job key with the job data.
+
+    - if delayed:
+      - computes timestamp.
+      - adds to delayed zset.
+      - Emits a global event 'delayed' if the job is delayed.
+    - if not delayed
+      - Adds the jobId to the wait/paused list in one of three ways:
+         - LIFO
+         - FIFO
+         - prioritized.
+      - Adds the job to the "added" list so that workers gets notified.
+
+    Input:
+      KEYS[1] 'wait',
+      KEYS[2] 'paused'
+      KEYS[3] 'meta'
+      KEYS[4] 'id'
+      KEYS[5] 'delayed'
+      KEYS[6] 'priority'
+      KEYS[7] 'completed'
+      KEYS[8] events stream key
+      KEYS[9] delay stream key
+
+      ARGV[1] msgpacked arguments array
+            [1]  key prefix,
+            [2]  custom id (will not generate one automatically)
+            [3]  name
+            [4]  timestamp
+            [5]  parentKey?
+            [6] waitChildrenKey key.
+            [7] parent dependencies key.
+
+      ARGV[2] Json stringified job data
+      ARGV[3] msgpacked options
+
+      Output:
+        jobId  - OK
+        -5     - Missing parent key
+]]
+local jobId
+local jobIdKey
+local rcall = redis.call
+
+local args = cmsgpack.unpack(ARGV[1])
+
+local data = ARGV[2]
+local opts = cmsgpack.unpack(ARGV[3])
+
+local parentKey = args[5]
+local parentId
+local parentQueueKey
+local parentData
+
+-- Includes
+--- @include "includes/destructureJobKey"
+
+if parentKey ~= nil then
+  if rcall("EXISTS", parentKey) ~= 1 then
+    return -5
+  end
+
+  parentId = getJobIdFromKey(parentKey)
+  parentQueueKey = getJobKeyPrefix(parentKey, ":" .. parentId)
+  local parent = {}
+  parent['id'] = parentId
+  parent['queueKey'] = parentQueueKey
+  parentData = cjson.encode(parent)
+end
+
+local jobCounter = rcall("INCR", KEYS[4])
+
+-- Includes
+--- @include "includes/updateParentDepsIfNeeded"
+
+local parentDependenciesKey = args[7]
+if args[2] == "" then
+  jobId = jobCounter
+  jobIdKey = args[1] .. jobId
+else
+  jobId = args[2]
+  jobIdKey = args[1] .. jobId
+  if rcall("EXISTS", jobIdKey) == 1 then
+    if parentKey ~= nil then
+      if rcall("ZSCORE", KEYS[7], jobId) ~= false then
+        local returnvalue = rcall("HGET", jobIdKey, "returnvalue")
+        updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey, parentId, jobIdKey, returnvalue)
+      else
+        if parentDependenciesKey ~= nil then
+          rcall("SADD", parentDependenciesKey, jobIdKey)
+        end
+      end
+      rcall("HMSET", jobIdKey, "parentKey", parentKey, "parent", parentData)
+    end
+    return jobId .. "" -- convert to string
+  end
+end
+
+-- Store the job.
+local jsonOpts = cjson.encode(opts)
+local delay = opts['delay'] or 0
+local priority = opts['priority'] or 0
+local timestamp = args[4]
+
+if parentKey ~= nil then
+  rcall("HMSET", jobIdKey, "name", args[3], "data", ARGV[2], "opts", jsonOpts,
+    "timestamp", timestamp, "delay", delay, "priority", priority, "parentKey", parentKey, "parent", parentData)
+else
+  rcall("HMSET", jobIdKey, "name", args[3], "data", ARGV[2], "opts", jsonOpts,
+    "timestamp", timestamp, "delay", delay, "priority", priority )
+end
+
+-- TODO: do not send data and opts to the event added (for performance reasons).
+rcall("XADD", KEYS[8], "*", "event", "added", "jobId", jobId, "name", args[3], "data", ARGV[2], "opts", jsonOpts)
+
+-- Check if job is delayed
+local delayedTimestamp = (delay > 0 and (timestamp + delay)) or 0
+
+-- Check if job is a parent, if so add to the parents set
+local waitChildrenKey = args[6]
+if waitChildrenKey ~= nil then
+    rcall("ZADD", waitChildrenKey, timestamp, jobId)
+    rcall("XADD", KEYS[8], "*", "event", "waiting-children", "jobId", jobId)
+elseif (delayedTimestamp ~= 0) then
+    local timestamp = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff)
+    rcall("ZADD", KEYS[5], timestamp, jobId)
+    rcall("XADD", KEYS[8], "*", "event", "delayed", "jobId", jobId, "delay",
+          delayedTimestamp)
+    rcall("XADD", KEYS[9], "*", "nextTimestamp", delayedTimestamp)
+else
+    local target
+
+    -- We check for the meta.paused key to decide if we are paused or not
+    -- (since an empty list and !EXISTS are not really the same)
+    local paused
+    if rcall("HEXISTS", KEYS[3], "paused") ~= 1 then
+        target = KEYS[1]
+        paused = false
+    else
+        target = KEYS[2]
+        paused = true
+    end
+
+    -- Standard or priority add
+    if priority == 0 then
+        -- LIFO or FIFO
+        local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH';
+        rcall(pushCmd, target, jobId)
+    else
+        -- Priority add
+        rcall("ZADD", KEYS[6], priority, jobId)
+        local count = rcall("ZCOUNT", KEYS[6], 0, priority)
+
+        local len = rcall("LLEN", target)
+        local id = rcall("LINDEX", target, len - (count - 1))
+        if id then
+            rcall("LINSERT", target, "BEFORE", id, jobId)
+        else
+            rcall("RPUSH", target, jobId)
+        end
+    end
+    -- Emit waiting event
+    rcall("XADD", KEYS[8], "*", "event", "waiting", "jobId", jobId)
+end
+
+-- Check if this job is a child of another job, if so add it to the parents dependencies
+-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
+-- fail in this case.
+if parentDependenciesKey ~= nil then
+    rcall("SADD", parentDependenciesKey, jobIdKey)
+end
+
+local maxEvents = rcall("HGET", KEYS[3], "opts.maxLenEvents")
+if (maxEvents) then rcall("XTRIM", KEYS[8], "MAXLEN", "~", maxEvents) end
+
+return jobId .. "" -- convert to string

package/dist/esm/commands/changeDelay-4.lua

@@ -0,0 +1,41 @@
+--[[
+  Change job delay when it is in delayed set.
+  Input:
+    KEYS[1] delayed key
+    KEYS[2] job key
+    KEYS[3] events stream
+    KEYS[4] delayed stream
+
+    ARGV[1] delayedTimestamp
+    ARGV[2] the id of the job
+  Output:
+    0 - OK
+   -1 - Missing job.
+   -3 - Job not in delayed set.
+
+  Events:
+    - delayed key.
+]]
+local rcall = redis.call
+
+if rcall("EXISTS", KEYS[2]) == 1 then
+
+  local jobId = ARGV[2]
+  local score = tonumber(ARGV[1])
+  local delayedTimestamp = (score / 0x1000)
+
+  local numRemovedElements = rcall("ZREM", KEYS[1], jobId)
+
+  if (numRemovedElements < 1)  then
+    return -3
+  end
+
+  rcall("ZADD", KEYS[1], score, jobId)
+
+  rcall("XADD", KEYS[3], "*", "event", "delayed", "jobId", jobId, "delay", delayedTimestamp);
+  rcall("XADD", KEYS[4], "*", "nextTimestamp", delayedTimestamp);
+
+  return 0
+else
+  return -1
+end

package/dist/esm/commands/cleanJobsInSet-2.lua

@@ -0,0 +1,65 @@
+--[[
+  Remove jobs from the specific set.
+
+  Input:
+    KEYS[1]  set key,
+    KEYS[2]  events stream key
+
+    ARGV[1]  jobId
+    ARGV[2]  timestamp
+    ARGV[3]  limit the number of jobs to be removed. 0 is unlimited
+    ARGV[4]  set name, can be any of 'wait', 'active', 'paused', 'delayed', 'completed', or 'failed'
+]]
+local rcall = redis.call
+local command = "ZRANGE"
+local isList = false
+
+if ARGV[4] == "wait" or ARGV[4] == "active" or ARGV[4] == "paused" then
+  command = "LRANGE"
+  isList = true
+end
+
+local rangeStart = 0
+local rangeEnd = -1
+
+local limit = tonumber(ARGV[3])
+
+-- If we're only deleting _n_ items, avoid retrieving all items
+-- for faster performance
+--
+-- Start from the tail of the list, since that's where oldest elements
+-- are generally added for FIFO lists
+if limit > 0 then
+  rangeStart = -1 - limit + 1
+  rangeEnd = -1
+end
+
+local jobs = rcall(command, KEYS[1], rangeStart, rangeEnd)
+local deleted = {}
+local deletedCount = 0
+local jobTS
+for _, job in ipairs(jobs) do
+  if limit > 0 and deletedCount >= limit then
+    break
+  end
+
+  local jobKey = ARGV[1] .. job
+  if (rcall("EXISTS", jobKey .. ":lock") == 0) then
+    jobTS = rcall("HGET", jobKey, "timestamp")
+    if (not jobTS or jobTS < ARGV[2]) then
+      if isList then
+        rcall("LREM", KEYS[1], 0, job)
+      else
+        rcall("ZREM", KEYS[1], job)
+      end
+      rcall("DEL", jobKey)
+      rcall("DEL", jobKey .. ":logs")
+      deletedCount = deletedCount + 1
+      table.insert(deleted, job)
+    end
+  end
+end
+
+rcall("XADD", KEYS[2], "*", "event", "cleaned", "count", deletedCount)
+
+return deleted

package/dist/esm/commands/drain-4.lua

@@ -0,0 +1,36 @@
+--[[
+  Drains the queue, removes all jobs that are waiting
+  or delayed, but not active, completed or failed
+
+  Input:
+    KEYS[1] 'wait',
+    KEYS[2] 'paused'
+    KEYS[3] 'delayed'
+    KEYS[4] 'priority'
+
+    ARGV[1]  queue key prefix
+]]
+local rcall = redis.call
+local queueBaseKey = ARGV[1]
+
+local function removeJobs (list)
+  for _, id in ipairs(list) do
+    rcall("DEL", queueBaseKey .. id)
+  end
+end
+
+local wait_ids = rcall("LRANGE", KEYS[1] , 0, -1)
+local paused_ids = rcall("LRANGE", KEYS[2] , 0, -1)
+
+removeJobs(wait_ids)
+removeJobs(paused_ids)
+
+if KEYS[3] ~= "" then
+  local delayed_ids = rcall("ZRANGE", KEYS[3] , 0, -1)
+  removeJobs(delayed_ids)
+  rcall("DEL", KEYS[3])
+end
+
+rcall("DEL", KEYS[1])
+rcall("DEL", KEYS[2])
+rcall("DEL", KEYS[4])

package/dist/esm/commands/extendLock-2.lua

@@ -0,0 +1,23 @@
+--[[
+  Extend lock and removes the job from the stalled set.
+
+  Input:
+    KEYS[1] 'lock',
+    KEYS[2] 'stalled'
+
+    ARGV[1]  token
+    ARGV[2]  lock duration in milliseconds
+    ARGV[3]  jobid
+
+  Output:
+    "1" if lock extented succesfully.
+]]
+local rcall = redis.call
+if rcall("GET", KEYS[1]) == ARGV[1] then
+  --   if rcall("SET", KEYS[1], ARGV[1], "PX", ARGV[2], "XX") then
+  if rcall("SET", KEYS[1], ARGV[1], "PX", ARGV[2]) then
+    rcall("SREM", KEYS[2], ARGV[3])
+    return 1
+  end
+end
+return 0

package/dist/esm/commands/getState-7.lua

@@ -0,0 +1,63 @@
+--[[
+  Get a job state
+
+  Input: 
+    KEYS[1] 'completed' key,
+    KEYS[2] 'failed' key
+    KEYS[3] 'delayed' key
+    KEYS[4] 'active' key
+    KEYS[5] 'wait' key
+    KEYS[6] 'paused' key
+    KEYS[7] waitChildrenKey key
+
+    ARGV[1] job id
+  Output:
+    'completed'
+    'failed'
+    'delayed'
+    'active'
+    'waiting'
+    'waiting-children'
+    'unknown'
+]]
+if redis.call("ZSCORE", KEYS[1], ARGV[1]) ~= false then
+  return "completed"
+end
+
+if redis.call("ZSCORE", KEYS[2], ARGV[1]) ~= false then
+  return "failed"
+end
+
+if redis.call("ZSCORE", KEYS[3], ARGV[1]) ~= false then
+  return "delayed"
+end
+
+local function item_in_list (list, item)
+  for _, v in pairs(list) do
+    if v == item then
+      return 1
+    end
+  end
+  return nil
+end
+
+local active_items = redis.call("LRANGE", KEYS[4] , 0, -1)
+if item_in_list(active_items, ARGV[1]) ~= nil then
+  return "active"
+end
+
+local wait_items = redis.call("LRANGE", KEYS[5] , 0, -1)
+if item_in_list(wait_items, ARGV[1]) ~= nil then
+  return "waiting"
+end
+
+local paused_items = redis.call("LRANGE", KEYS[6] , 0, -1)
+if item_in_list(paused_items, ARGV[1]) ~= nil then
+  return "waiting"
+end
+
+if redis.call("ZSCORE", KEYS[7], ARGV[1]) ~= false then
+  return "waiting-children"
+end
+
+return "unknown"

package/dist/esm/commands/getStateV2-7.lua

@@ -0,0 +1,51 @@
+--[[
+  Get a job state
+
+  Input: 
+    KEYS[1] 'completed' key,
+    KEYS[2] 'failed' key
+    KEYS[3] 'delayed' key
+    KEYS[4] 'active' key
+    KEYS[5] 'wait' key
+    KEYS[6] 'paused' key
+    KEYS[7] waitChildrenKey key
+
+    ARGV[1] job id
+  Output:
+    'completed'
+    'failed'
+    'delayed'
+    'active'
+    'waiting'
+    'waiting-children'
+    'unknown'
+]]
+if redis.call("ZSCORE", KEYS[1], ARGV[1]) ~= false then
+  return "completed"
+end
+
+if redis.call("ZSCORE", KEYS[2], ARGV[1]) ~= false then
+  return "failed"
+end
+
+if redis.call("ZSCORE", KEYS[3], ARGV[1]) ~= false then
+  return "delayed"
+end
+
+if redis.call("LPOS", KEYS[4] , ARGV[1]) ~= false then
+  return "active"
+end
+
+if redis.call("LPOS", KEYS[5] , ARGV[1]) ~= false then
+  return "waiting"
+end
+
+if redis.call("LPOS", KEYS[6] , ARGV[1]) ~= false then
+  return "waiting"
+end
+
+if redis.call("ZSCORE", KEYS[7] , ARGV[1]) ~= false then
+  return "waiting-children"
+end
+
+return "unknown"

package/dist/esm/commands/includes/destructureJobKey.lua

@@ -0,0 +1,12 @@
+--[[
+  Functions to destructure job key.
+  Just a bit of warning, these functions may be a bit slow and affect performance significantly.
+]]
+
+local getJobIdFromKey = function (jobKey)
+  return string.match(jobKey, ".*:(.*)")
+end
+
+local getJobKeyPrefix = function (jobKey, jobId)
+  return string.sub(jobKey, 0, #jobKey - #jobId)
+end

package/dist/esm/commands/includes/updateParentDepsIfNeeded.lua

@@ -0,0 +1,19 @@
+--[[
+  Validate and move or add dependencies to parent.
+]]
+
+local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey, parentId, jobIdKey, returnvalue )
+  local processedSet = parentKey .. ":processed"
+  rcall("HSET", processedSet, jobIdKey, returnvalue)
+  local activeParent = rcall("ZSCORE", parentQueueKey .. ":waiting-children", parentId)
+  if rcall("SCARD", parentDependenciesKey) == 0 and activeParent then 
+    rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
+    if rcall("HEXISTS", parentQueueKey .. ":meta", "paused") ~= 1 then
+      rcall("RPUSH", parentQueueKey .. ":wait", parentId)
+    else
+      rcall("RPUSH", parentQueueKey .. ":paused", parentId)
+    end
+    local parentEventStream = parentQueueKey .. ":events"
+    rcall("XADD", parentEventStream, "*", "event", "active", "jobId", parentId, "prev", "waiting-children")
+  end
+end

package/dist/esm/commands/isFinished-3.lua

@@ -0,0 +1,48 @@
+--[[
+  Checks if a job is finished (.i.e. is in the completed or failed set)
+
+  Input: 
+    KEYS[1] completed key
+    KEYS[2] failed key
+    KEYS[3] job key
+
+    ARGV[1] job id
+    ARGV[2] return value?
+  Output:
+    0 - Not finished.
+    1 - Completed.
+    2 - Failed.
+   -5 - Missing job. 
+]]
+local rcall = redis.call
+if rcall("EXISTS", KEYS[3]) ~= 1 then
+  if ARGV[2] == "1" then
+
+    return {-5,"Missing key for job " .. KEYS[3] .. ". isFinished"}
+  end  
+  return -5
+end
+
+if rcall("ZSCORE", KEYS[1], ARGV[1]) ~= false then
+  if ARGV[2] == "1" then
+    local returnValue = rcall("HGET", KEYS[3], "returnvalue")
+
+    return {1,returnValue}
+  end
+  return 1
+end
+
+if rcall("ZSCORE", KEYS[2], ARGV[1]) ~= false then
+  if ARGV[2] == "1" then
+    local failedReason = rcall("HGET", KEYS[3], "failedReason")
+
+    return {2,failedReason}
+  end
+  return 2
+end
+
+if ARGV[2] == "1" then
+  return {0}
+end
+
+return 0

package/dist/esm/commands/isJobInList-1.lua

@@ -0,0 +1,20 @@
+--[[
+      Checks if job is in a given list.
+
+      Input:
+        KEYS[1]
+        ARGV[1]
+
+      Output:
+        1 if element found in the list.
+]]
+local function item_in_list (list, item)
+  for _, v in pairs(list) do
+    if v == item then
+      return 1
+    end
+  end
+  return nil
+end
+local items = redis.call("LRANGE", KEYS[1] , 0, -1)
+return item_in_list(items, ARGV[1])

package/dist/esm/commands/moveStalledJobsToWait-8.lua

@@ -0,0 +1,112 @@
+--[[
+  Move stalled jobs to wait.
+
+    Input:
+      KEYS[1] 'stalled' (SET)
+      KEYS[2] 'wait',   (LIST)
+      KEYS[3] 'active', (LIST)
+      KEYS[4] 'failed', (ZSET)
+      KEYS[5] 'stalled-check', (KEY)
+
+      KEYS[6] 'meta', (KEY)
+      KEYS[7] 'paused', (LIST)
+
+      KEYS[8] 'event stream' (STREAM)
+
+      ARGV[1]  Max stalled job count
+      ARGV[2]  queue.toKey('')
+      ARGV[3]  timestamp
+      ARGV[4]  max check time
+
+    Events:
+      'stalled' with stalled job id.
+]]
+local rcall = redis.call
+
+local function batches(n, batchSize)
+  local i = 0
+
+  return function()
+    local from = i * batchSize + 1
+    i = i + 1
+    if (from <= n) then
+      local to = math.min(from + batchSize - 1, n)
+      return from, to
+    end
+  end
+end
+
+-- Check if we need to check for stalled jobs now.
+if rcall("EXISTS", KEYS[5]) == 1 then return {{}, {}} end
+
+rcall("SET", KEYS[5], ARGV[3], "PX", ARGV[4])
+
+-- Move all stalled jobs to wait
+local stalling = rcall('SMEMBERS', KEYS[1])
+local stalled = {}
+local failed = {}
+if (#stalling > 0) then
+
+    local dst
+    -- wait or paused destination
+    if rcall("HEXISTS", KEYS[6], "paused") ~= 1 then
+        dst = KEYS[2]
+    else
+        dst = KEYS[7]
+    end
+
+    rcall('DEL', KEYS[1])
+
+    local MAX_STALLED_JOB_COUNT = tonumber(ARGV[1])
+
+    -- Remove from active list
+    for i, jobId in ipairs(stalling) do
+        local jobKey = ARGV[2] .. jobId
+
+        -- Check that the lock is also missing, then we can handle this job as really stalled.
+        if (rcall("EXISTS", jobKey .. ":lock") == 0) then
+            --  Remove from the active queue.
+            local removed = rcall("LREM", KEYS[3], 1, jobId)
+
+            if (removed > 0) then
+                -- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
+                local stalledCount = rcall("HINCRBY", jobKey, "stalledCounter",
+                                           1)
+                if (stalledCount > MAX_STALLED_JOB_COUNT) then
+                    local failedReason = "job stalled more than allowable limit" 
+                    rcall("ZADD", KEYS[4], ARGV[3], jobId)
+                    rcall("HMSET", jobKey, "failedReason",
+                          failedReason, "finishedOn", ARGV[3])
+                    rcall("XADD", KEYS[8], "*", "event", "failed", "jobId",
+                          jobId, 'prev', 'active', 'failedReason',
+                          failedReason)
+                    table.insert(failed, jobId)
+                else
+                    -- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
+                    rcall("RPUSH", dst, jobId)
+                    rcall("XADD", KEYS[8], "*", "event", "waiting", "jobId",
+                          jobId, 'prev', 'active')
+
+                    -- Emit the stalled event
+                    rcall("XADD", KEYS[8], "*", "event", "stalled", "jobId",
+                          jobId)
+                    table.insert(stalled, jobId)
+                end
+            end
+        end
+    end
+end
+
+-- Mark potentially stalled jobs
+local active = rcall('LRANGE', KEYS[3], 0, -1)
+
+if (#active > 0) then
+  for from, to in batches(#active, 7000) do
+    rcall('SADD', KEYS[1], unpack(active, from, to))
+  end
+end
+
+local maxEvents = rcall("HGET", KEYS[6], "opts.maxLenEvents")
+if maxEvents then rcall("XTRIM", KEYS[8], "MAXLEN", "~", maxEvents) end
+
+return {failed, stalled}