bullmq 4.0.0-beta.1 → 4.0.0

package/dist/esm/commands/addJob-9.lua

@@ -0,0 +1,170 @@
+--[[
+  Adds a job to the queue by doing the following:
+    - Increases the job counter if needed.
+    - Creates a new job key with the job data.
+
+    - if delayed:
+      - computes timestamp.
+      - adds to delayed zset.
+      - Emits a global event 'delayed' if the job is delayed.
+    - if not delayed
+      - Adds the jobId to the wait/paused list in one of three ways:
+         - LIFO
+         - FIFO
+         - prioritized.
+      - Adds the job to the "added" list so that workers gets notified.
+
+    Input:
+      KEYS[1] 'wait',
+      KEYS[2] 'paused'
+      KEYS[3] 'meta'
+      KEYS[4] 'id'
+      KEYS[5] 'delayed'
+      KEYS[6] 'prioritized'
+      KEYS[7] 'completed'
+      KEYS[8] events stream key
+      KEYS[9] 'pc' priority counter
+
+      ARGV[1] msgpacked arguments array
+            [1]  key prefix,
+            [2]  custom id (will not generate one automatically)
+            [3]  name
+            [4]  timestamp
+            [5]  parentKey?
+            [6]  waitChildrenKey key.
+            [7]  parent dependencies key.
+            [8]  parent? {id, queueKey}
+            [9]  repeat job key
+            
+      ARGV[2] Json stringified job data
+      ARGV[3] msgpacked options
+
+      Output:
+        jobId  - OK
+        -5     - Missing parent key
+]]
+local jobId
+local jobIdKey
+local rcall = redis.call
+
+local args = cmsgpack.unpack(ARGV[1])
+
+local data = ARGV[2]
+local opts = cmsgpack.unpack(ARGV[3])
+
+local parentKey = args[5]
+local repeatJobKey = args[9]
+local parent = args[8]
+local parentData
+
+-- Includes
+--- @include "includes/addDelayMarkerIfNeeded"
+--- @include "includes/addJobWithPriority"
+--- @include "includes/getTargetQueueList"
+--- @include "includes/trimEvents"
+--- @include "includes/getNextDelayedTimestamp"
+--- @include "includes/updateParentDepsIfNeeded"
+
+if parentKey ~= nil then
+  if rcall("EXISTS", parentKey) ~= 1 then
+    return -5
+  end
+
+  parentData = cjson.encode(parent)
+end
+
+local jobCounter = rcall("INCR", KEYS[4])
+
+-- Trim events before emiting them to avoid trimming events emitted in this script
+trimEvents(KEYS[3], KEYS[8])
+
+local parentDependenciesKey = args[7]
+local timestamp = args[4]
+if args[2] == "" then
+  jobId = jobCounter
+  jobIdKey = args[1] .. jobId
+else
+  jobId = args[2]
+  jobIdKey = args[1] .. jobId
+  if rcall("EXISTS", jobIdKey) == 1 then
+    if parentKey ~= nil then
+      if rcall("ZSCORE", KEYS[7], jobId) ~= false then
+        local returnvalue = rcall("HGET", jobIdKey, "returnvalue")
+        updateParentDepsIfNeeded(parentKey, parent['queueKey'], parentDependenciesKey,
+          parent['id'], jobIdKey, returnvalue, timestamp)
+      else
+        if parentDependenciesKey ~= nil then
+          rcall("SADD", parentDependenciesKey, jobIdKey)
+        end
+      end
+      rcall("HMSET", jobIdKey, "parentKey", parentKey, "parent", parentData)
+    end
+    rcall("XADD", KEYS[8], "*", "event", "duplicated", "jobId", jobId)
+
+    return jobId .. "" -- convert to string
+  end
+end
+
+-- Store the job.
+local jsonOpts = cjson.encode(opts)
+local delay = opts['delay'] or 0
+local priority = opts['priority'] or 0
+
+local optionalValues = {}
+if parentKey ~= nil then
+  table.insert(optionalValues, "parentKey")
+  table.insert(optionalValues, parentKey)
+  table.insert(optionalValues, "parent")
+  table.insert(optionalValues, parentData)
+end
+
+if repeatJobKey ~= nil then
+  table.insert(optionalValues, "rjk")
+  table.insert(optionalValues, repeatJobKey)
+end
+
+rcall("HMSET", jobIdKey, "name", args[3], "data", ARGV[2], "opts", jsonOpts,
+  "timestamp", timestamp, "delay", delay, "priority", priority, unpack(optionalValues))
+
+rcall("XADD", KEYS[8], "*", "event", "added", "jobId", jobId, "name", args[3])
+
+-- Check if job is delayed
+local delayedTimestamp = (delay > 0 and (timestamp + delay)) or 0
+
+-- Check if job is a parent, if so add to the parents set
+local waitChildrenKey = args[6]
+if waitChildrenKey ~= nil then
+  rcall("ZADD", waitChildrenKey, timestamp, jobId)
+  rcall("XADD", KEYS[8], "*", "event", "waiting-children", "jobId", jobId)
+elseif (delayedTimestamp ~= 0) then
+  local score = delayedTimestamp * 0x1000 + bit.band(jobCounter, 0xfff)
+  rcall("ZADD", KEYS[5], score, jobId)
+  rcall("XADD", KEYS[8], "*", "event", "delayed", "jobId", jobId, "delay",
+        delayedTimestamp)
+  -- If wait list is empty, and this delayed job is the next one to be processed,
+  -- then we need to signal the workers by adding a dummy job (jobId 0:delay) to the wait list.
+  local target = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])
+  addDelayMarkerIfNeeded(target, KEYS[5])
+else
+  local target, paused = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])
+
+  -- Standard or priority add
+  if priority == 0 then
+    -- LIFO or FIFO
+    local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH'
+    rcall(pushCmd, target, jobId)
+  else
+    addJobWithPriority(KEYS[1], KEYS[6], priority, paused, jobId, KEYS[9])
+  end
+  -- Emit waiting event
+  rcall("XADD", KEYS[8], "*", "event", "waiting", "jobId", jobId)
+end
+
+-- Check if this job is a child of another job, if so add it to the parents dependencies
+-- TODO: Should not be possible to add a child job to a parent that is not in the "waiting-children" status.
+-- fail in this case.
+if parentDependenciesKey ~= nil then
+  rcall("SADD", parentDependenciesKey, jobIdKey)
+end
+
+return jobId .. "" -- convert to string

package/dist/esm/commands/changeDelay-3.lua

@@ -0,0 +1,40 @@
+--[[
+  Change job delay when it is in delayed set.
+  Input:
+    KEYS[1] delayed key
+    KEYS[2] job key
+    KEYS[3] events stream
+
+    ARGV[1] delay
+    ARGV[2] delayedTimestamp
+    ARGV[3] the id of the job
+  Output:
+    0 - OK
+   -1 - Missing job.
+   -3 - Job not in delayed set.
+
+  Events:
+    - delayed key.
+]]
+local rcall = redis.call
+
+if rcall("EXISTS", KEYS[2]) == 1 then
+  local jobId = ARGV[3]
+  local score = tonumber(ARGV[2])
+  local delayedTimestamp = (score / 0x1000)
+
+  local numRemovedElements = rcall("ZREM", KEYS[1], jobId)
+
+  if numRemovedElements < 1 then
+    return -3
+  end
+
+  rcall("HSET", KEYS[2], "delay", tonumber(ARGV[1]))
+  rcall("ZADD", KEYS[1], score, jobId)
+
+  rcall("XADD", KEYS[3], "*", "event", "delayed", "jobId", jobId, "delay", delayedTimestamp)
+
+  return 0
+else
+  return -1
+end

package/dist/esm/commands/changePriority-5.lua

@@ -0,0 +1,52 @@
+--[[
+  Change job priority
+  Input:
+    KEYS[1] 'wait',
+    KEYS[2] 'paused'
+    KEYS[3] 'meta'
+    KEYS[4] 'prioritized'
+    KEYS[5] 'pc' priority counter
+
+    ARGV[1] priority value
+    ARGV[2] job key
+    ARGV[3] job id
+    ARGV[4] lifo
+
+    Output:
+       0  - OK
+      -1  - Missing job
+]]
+local jobKey = ARGV[2]
+local jobId = ARGV[3]
+local priority = tonumber(ARGV[1])
+local rcall = redis.call
+
+-- Includes
+--- @include "includes/addJobWithPriority"
+--- @include "includes/getTargetQueueList"
+
+if rcall("EXISTS", jobKey) == 1 then
+  local target, paused = getTargetQueueList(KEYS[3], KEYS[1], KEYS[2])
+
+  if rcall("ZREM", KEYS[4], jobId) > 0 then
+    addJobWithPriority(KEYS[1], KEYS[4], priority, paused, jobId, KEYS[5])
+  else
+    local numRemovedElements = rcall("LREM", target, -1, jobId)
+    if numRemovedElements > 0 then
+      -- Standard or priority add
+      if priority == 0 then
+        -- LIFO or FIFO
+        local pushCmd = ARGV[4] == '1' and 'RPUSH' or 'LPUSH';
+        rcall(pushCmd, target, jobId)
+      else
+        addJobWithPriority(KEYS[1], KEYS[4], priority, paused, jobId, KEYS[5])
+      end
+    end
+  end
+
+  rcall("HSET", jobKey, "priority", priority)
+
+  return 0
+else
+  return -1
+end

package/dist/esm/commands/cleanJobsInSet-2.lua

@@ -0,0 +1,48 @@
+--[[
+  Remove jobs from the specific set.
+
+  Input:
+    KEYS[1]  set key,
+    KEYS[2]  events stream key
+
+    ARGV[1]  jobKey prefix
+    ARGV[2]  timestamp
+    ARGV[3]  limit the number of jobs to be removed. 0 is unlimited
+    ARGV[4]  set name, can be any of 'wait', 'active', 'paused', 'delayed', 'completed', or 'failed'
+]]
+local rcall = redis.call
+local rangeStart = 0
+local rangeEnd = -1
+
+local limit = tonumber(ARGV[3])
+
+-- If we're only deleting _n_ items, avoid retrieving all items
+-- for faster performance
+--
+-- Start from the tail of the list, since that's where oldest elements
+-- are generally added for FIFO lists
+if limit > 0 then
+  rangeStart = -1 - limit + 1
+  rangeEnd = -1
+end
+
+-- Includes
+--- @include "includes/cleanList"
+--- @include "includes/cleanSet"
+
+local result
+if ARGV[4] == "active" then
+  result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], false)
+elseif ARGV[4] == "delayed" then
+  result = cleanSet(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], limit, {"processedOn", "timestamp"})
+elseif ARGV[4] == "prioritized" then
+  result = cleanSet(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], limit, {"timestamp"})
+elseif ARGV[4] == "wait" or ARGV[4] == "paused" then
+  result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], true)
+else
+  result = cleanSet(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], limit, {"finishedOn"} )
+end
+
+rcall("XADD", KEYS[2], "*", "event", "cleaned", "count", result[2])
+
+return result[1]

package/dist/esm/commands/drain-4.lua

@@ -0,0 +1,26 @@
+--[[
+  Drains the queue, removes all jobs that are waiting
+  or delayed, but not active, completed or failed
+
+  Input:
+    KEYS[1] 'wait',
+    KEYS[2] 'paused'
+    KEYS[3] 'delayed'
+    KEYS[4] 'prioritized'
+
+    ARGV[1]  queue key prefix
+]]
+local rcall = redis.call
+local queueBaseKey = ARGV[1]
+
+--- @include "includes/removeJobs"
+--- @include "includes/removeZSetJobs"
+
+removeListJobs(KEYS[1], true, queueBaseKey, 0) --wait
+removeListJobs(KEYS[2], true, queueBaseKey, 0) --paused
+
+if KEYS[3] ~= "" then
+  removeZSetJobs(KEYS[3], true, queueBaseKey, 0) --delayed
+end
+
+removeZSetJobs(KEYS[4], true, queueBaseKey, 0) --prioritized

package/{src → dist/esm}/commands/extendLock-2.lua

@@ -14,6 +14,7 @@
 ]]
 local rcall = redis.call
 if rcall("GET", KEYS[1]) == ARGV[1] then
+  --   if rcall("SET", KEYS[1], ARGV[1], "PX", ARGV[2], "XX") then
   if rcall("SET", KEYS[1], ARGV[1], "PX", ARGV[2]) then
     rcall("SREM", KEYS[2], ARGV[3])
     return 1

package/dist/esm/commands/getCounts-1.lua

@@ -0,0 +1,35 @@
+--[[
+  Get counts per provided states
+
+    Input:
+      KEYS[1]    'prefix'
+
+      ARGV[1...] types
+]]
+local rcall = redis.call;
+local prefix = KEYS[1]
+local results = {}
+
+for i = 1, #ARGV do
+  local stateKey = prefix .. ARGV[i]
+  if ARGV[i] == "wait" or ARGV[i] == "paused" then
+    local marker = rcall("LINDEX", stateKey, -1)
+    if marker and string.sub(marker, 1, 2) == "0:" then
+      local count = rcall("LLEN", stateKey)
+      if count > 1 then
+        rcall("RPOP", stateKey)
+        results[#results+1] = count-1
+      else
+        results[#results+1] = 0
+      end
+    else
+      results[#results+1] = rcall("LLEN", stateKey)
+    end
+  elseif ARGV[i] == "active" then
+    results[#results+1] = rcall("LLEN", stateKey)
+  else
+    results[#results+1] = rcall("ZCARD", stateKey)
+  end
+end
+
+return results

package/dist/esm/commands/getRanges-1.lua

@@ -0,0 +1,69 @@
+--[[
+  Get job ids per provided states
+
+    Input:
+      KEYS[1]    'prefix'
+
+      ARGV[1]    start
+      ARGV[2]    end
+      ARGV[3]    asc
+      ARGV[4...] types
+]]
+local rcall = redis.call
+local prefix = KEYS[1]
+local rangeStart = tonumber(ARGV[1])
+local rangeEnd = tonumber(ARGV[2])
+local asc = ARGV[3]
+local results = {}
+
+local function getRangeInList(listKey, asc, rangeStart, rangeEnd, results)
+  if asc == "1" then
+    local modifiedRangeStart
+    local modifiedRangeEnd
+    if rangeStart == -1 then
+      modifiedRangeStart = 0
+    else
+      modifiedRangeStart = -(rangeStart + 1)
+    end
+
+    if rangeEnd == -1 then
+      modifiedRangeEnd = 0
+    else
+      modifiedRangeEnd = -(rangeEnd + 1)
+    end
+
+    results[#results+1] = rcall("LRANGE", listKey,
+      modifiedRangeEnd,
+      modifiedRangeStart)
+  else
+    results[#results+1] = rcall("LRANGE", listKey, rangeStart, rangeEnd)
+  end
+end
+
+for i = 4, #ARGV do
+  local stateKey = prefix .. ARGV[i]
+  if ARGV[i] == "wait" or ARGV[i] == "paused" then
+    local marker = rcall("LINDEX", stateKey, -1)
+    if marker and string.sub(marker, 1, 2) == "0:" then
+      local count = rcall("LLEN", stateKey)
+      if count > 1 then
+        rcall("RPOP", stateKey)
+        getRangeInList(stateKey, asc, rangeStart, rangeEnd, results)
+      else
+        results[#results+1] = {}
+      end
+    else
+      getRangeInList(stateKey, asc, rangeStart, rangeEnd, results)
+    end
+  elseif ARGV[i] == "active" then
+    getRangeInList(stateKey, asc, rangeStart, rangeEnd, results)
+  else
+    if asc == "1" then
+      results[#results+1] = rcall("ZRANGE", stateKey, rangeStart, rangeEnd)
+    else
+      results[#results+1] = rcall("ZREVRANGE", stateKey, rangeStart, rangeEnd)
+    end
+  end
+end
+
+return results

package/dist/esm/commands/getState-8.lua

@@ -0,0 +1,66 @@
+--[[
+  Get a job state
+
+  Input: 
+    KEYS[1] 'completed' key,
+    KEYS[2] 'failed' key
+    KEYS[3] 'delayed' key
+    KEYS[4] 'active' key
+    KEYS[5] 'wait' key
+    KEYS[6] 'paused' key
+    KEYS[7] 'waiting-children' key
+    KEYS[8] 'prioritized' key
+
+    ARGV[1] job id
+    ARGV[2] job key
+  Output:
+    'completed'
+    'failed'
+    'delayed'
+    'active'
+    'prioritized'
+    'waiting'
+    'waiting-children'
+    'unknown'
+]]
+local rcall = redis.call
+
+if rcall("ZSCORE", KEYS[1], ARGV[1]) ~= false then
+  return "completed"
+end
+
+if rcall("ZSCORE", KEYS[2], ARGV[1]) ~= false then
+  return "failed"
+end
+
+if rcall("ZSCORE", KEYS[3], ARGV[1]) ~= false then
+  return "delayed"
+end
+
+if rcall("ZSCORE", KEYS[8], ARGV[1]) ~= false then
+  return "prioritized"
+end
+
+-- Includes
+--- @include "includes/checkItemInList"
+
+local active_items = rcall("LRANGE", KEYS[4] , 0, -1)
+if checkItemInList(active_items, ARGV[1]) ~= nil then
+  return "active"
+end
+
+local wait_items = rcall("LRANGE", KEYS[5] , 0, -1)
+if checkItemInList(wait_items, ARGV[1]) ~= nil then
+  return "waiting"
+end
+
+local paused_items = rcall("LRANGE", KEYS[6] , 0, -1)
+if checkItemInList(paused_items, ARGV[1]) ~= nil then
+  return "waiting"
+end
+
+if rcall("ZSCORE", KEYS[7], ARGV[1]) ~= false then
+  return "waiting-children"
+end
+
+return "unknown"

package/dist/esm/commands/getStateV2-8.lua

@@ -0,0 +1,59 @@
+--[[
+  Get a job state
+
+  Input: 
+    KEYS[1] 'completed' key,
+    KEYS[2] 'failed' key
+    KEYS[3] 'delayed' key
+    KEYS[4] 'active' key
+    KEYS[5] 'wait' key
+    KEYS[6] 'paused' key
+    KEYS[7] 'waiting-children' key
+    KEYS[8] 'prioritized' key
+
+    ARGV[1] job id
+    ARGV[2] job key
+  Output:
+    'completed'
+    'failed'
+    'delayed'
+    'active'
+    'waiting'
+    'waiting-children'
+    'unknown'
+]]
+local rcall = redis.call
+
+if rcall("ZSCORE", KEYS[1], ARGV[1]) ~= false then
+  return "completed"
+end
+
+if rcall("ZSCORE", KEYS[2], ARGV[1]) ~= false then
+  return "failed"
+end
+
+if rcall("ZSCORE", KEYS[3], ARGV[1]) ~= false then
+  return "delayed"
+end
+
+if rcall("ZSCORE", KEYS[8], ARGV[1]) ~= false then
+  return "prioritized"
+end
+
+if rcall("LPOS", KEYS[4] , ARGV[1]) ~= false then
+  return "active"
+end
+
+if rcall("LPOS", KEYS[5] , ARGV[1]) ~= false then
+  return "waiting"
+end
+
+if rcall("LPOS", KEYS[6] , ARGV[1]) ~= false then
+  return "waiting"
+end
+
+if rcall("ZSCORE", KEYS[7] , ARGV[1]) ~= false then
+  return "waiting-children"
+end
+
+return "unknown"

package/dist/esm/commands/includes/addDelayMarkerIfNeeded.lua

@@ -0,0 +1,15 @@
+--[[
+  Add delay marker if needed.
+]]
+
+-- Includes
+--- @include "getNextDelayedTimestamp"
+
+local function addDelayMarkerIfNeeded(targetKey, delayedKey)
+  if rcall("LLEN", targetKey) == 0 then
+    local nextTimestamp = getNextDelayedTimestamp(delayedKey)
+    if nextTimestamp ~= nil then
+      rcall("LPUSH", targetKey, "0:" .. nextTimestamp)
+    end
+  end
+end

package/dist/esm/commands/includes/addJobWithPriority.lua

@@ -0,0 +1,15 @@
+--[[
+  Function to add job considering priority.
+]]
+
+-- Includes
+--- @include "addPriorityMarkerIfNeeded"
+
+local function addJobWithPriority(waitKey, prioritizedKey, priority, paused, jobId, priorityCounterKey)
+  local prioCounter = rcall("INCR", priorityCounterKey)
+  local score = priority * 0x100000000 + bit.band(prioCounter, 0xffffffffffff)
+  rcall("ZADD", prioritizedKey, score, jobId)
+  if not paused then
+    addPriorityMarkerIfNeeded(waitKey)
+  end
+end

package/dist/esm/commands/includes/addPriorityMarkerIfNeeded.lua

@@ -0,0 +1,12 @@
+--[[
+  Function priority marker to wait if needed
+  in order to wake up our workers and to respect priority
+  order as much as possible
+]]
+local function addPriorityMarkerIfNeeded(waitKey)
+  local waitLen = rcall("LLEN", waitKey)
+
+  if waitLen == 0 then
+    rcall("LPUSH", waitKey, "0:0")
+  end
+end

package/dist/esm/commands/includes/batches.lua

@@ -0,0 +1,18 @@
+--[[
+  Function to loop in batches.
+  Just a bit of warning, some commands as ZREM
+  could receive a maximum of 7000 parameters per call.
+]]
+
+local function batches(n, batchSize)
+  local i = 0
+
+  return function()
+    local from = i * batchSize + 1
+    i = i + 1
+    if (from <= n) then
+      local to = math.min(from + batchSize - 1, n)
+      return from, to
+    end
+  end
+end

package/dist/esm/commands/includes/checkItemInList.lua

@@ -0,0 +1,12 @@
+--[[
+  Functions to check if a item belongs to a list.
+]]
+
+local function checkItemInList(list, item)
+  for _, v in pairs(list) do
+    if v == item then
+      return 1
+    end
+  end
+  return nil
+end