bullmq 2.2.0 → 2.3.0

package/dist/esm/commands/includes/cleanList.lua

@@ -1,50 +0,0 @@
---[[
-  Function to clean job list.
-  Returns jobIds and deleted count number.
-]]
-
--- Includes
---- @include "getTimestamp"
---- @include "removeJob"
-
-local function cleanList(listKey, jobKeyPrefix, rangeStart, rangeEnd,
-  timestamp, isWaiting)
-  local jobs = rcall("LRANGE", listKey, rangeStart, rangeEnd)
-  local deleted = {}
-  local deletedCount = 0
-  local jobTS
-  local deletionMarker = ''
-  local jobIdsLen = #jobs
-  for i, job in ipairs(jobs) do
-    if limit > 0 and deletedCount >= limit then
-      break
-    end
-  
-    local jobKey = jobKeyPrefix .. job
-    if (isWaiting or rcall("EXISTS", jobKey .. ":lock") == 0) then
-      -- Find the right timestamp of the job to compare to maxTimestamp:
-      -- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
-      -- * processedOn represents when the job was last attempted, but it doesn't get populated until
-      --   the job is first tried
-      -- * timestamp is the original job submission time
-      -- Fetch all three of these (in that order) and use the first one that is set so that we'll leave jobs
-      -- that have been active within the grace period:
-      jobTS = getTimestamp(jobKey, {"finishedOn", "processedOn", "timestamp"})
-      if (not jobTS or jobTS < timestamp) then
-        -- replace the entry with a deletion marker; the actual deletion will
-        -- occur at the end of the script
-        rcall("LSET", listKey, rangeEnd - jobIdsLen + i, deletionMarker)
-        removeJob(job, true, jobKeyPrefix)
-        if isWaiting then
-          rcall("ZREM", jobKeyPrefix .. "priority", job)
-        end
-        deletedCount = deletedCount + 1
-        table.insert(deleted, job)
-      end
-    end
-  end
-
-  rcall("LREM", listKey, 0, deletionMarker)
-
-  return {deleted, deletedCount}
-end

package/dist/esm/commands/includes/cleanSet.lua

@@ -1,50 +0,0 @@
---[[
-  Function to clean job set.
-  Returns jobIds and deleted count number.
-]]
-
--- Includes
---- @include "batches"
---- @include "getTimestamp"
---- @include "removeJob"
-
--- We use ZRANGEBYSCORE to make the case where we're deleting a limited number
--- of items in a sorted set only run a single iteration. If we simply used
--- ZRANGE, we may take a long time traversing through jobs that are within the
--- grace period.
-local function getJobs(setKey, rangeStart, rangeEnd, maxTimestamp, limit)
-  if limit > 0 then
-    return rcall("ZRANGEBYSCORE", setKey, 0, maxTimestamp, "LIMIT", 0, limit)
-  else
-    return rcall("ZRANGE", setKey, rangeStart, rangeEnd)
-  end
-end
-
-local function cleanSet(setKey, jobKeyPrefix, rangeStart, rangeEnd, timestamp, limit, attributes)
-  local jobs = getJobs(setKey, rangeStart, rangeEnd, timestamp, limit)
-  local deleted = {}
-  local deletedCount = 0
-  local jobTS
-  for i, job in ipairs(jobs) do
-    if limit > 0 and deletedCount >= limit then
-      break
-    end
-  
-    local jobKey = jobKeyPrefix .. job
-    -- * finishedOn says when the job was completed, but it isn't set unless the job has actually completed
-    jobTS = getTimestamp(jobKey, attributes)
-    if (not jobTS or jobTS < timestamp) then
-      removeJob(job, true, jobKeyPrefix)
-      deletedCount = deletedCount + 1
-      table.insert(deleted, job)
-    end
-  end
-
-  if(#deleted > 0) then
-    for from, to in batches(#deleted, 7000) do
-      rcall("ZREM", setKey, unpack(deleted, from, to))
-    end
-  end
-  
-  return {deleted, deletedCount}
-end

package/dist/esm/commands/includes/collectMetrics.lua

@@ -1,46 +0,0 @@
---[[
-  Functions to collect metrics based on a current and previous count of jobs.
-  Granualarity is fixed at 1 minute.
-]] 
---- @include "batches"
-local function collectMetrics(metaKey, dataPointsList, maxDataPoints,
-                                 timestamp)
-    -- Increment current count
-    local count = rcall("HINCRBY", metaKey, "count", 1) - 1
-
-    -- Compute how many data points we need to add to the list, N.
-    local prevTS = rcall("HGET", metaKey, "prevTS")
-
-    if not prevTS then
-        -- If prevTS is nil, set it to the current timestamp
-        rcall("HSET", metaKey, "prevTS", timestamp, "prevCount", 0)
-        return
-    end
-
-    local N = math.floor((timestamp - prevTS) / 60000)
-
-    if N > 0 then
-        local delta = count - rcall("HGET", metaKey, "prevCount")
-        -- If N > 1, add N-1 zeros to the list
-        if N > 1 then
-            local points = {}
-            points[1] = delta
-            for i = 2, N do
-                points[i] = 0
-            end
-
-            for from, to in batches(#points, 7000) do
-                rcall("LPUSH", dataPointsList, unpack(points, from, to))
-            end
-        else
-            -- LPUSH delta to the list
-            rcall("LPUSH", dataPointsList, delta)
-        end
-
-        -- LTRIM to keep list to its max size
-        rcall("LTRIM", dataPointsList, 0, maxDataPoints - 1)
-
-        -- update prev count with current count
-        rcall("HSET", metaKey, "prevCount", count, "prevTS", timestamp)
-    end
-end

package/dist/esm/commands/includes/destructureJobKey.lua

@@ -1,12 +0,0 @@
---[[
-  Functions to destructure job key.
-  Just a bit of warning, these functions may be a bit slow and affect performance significantly.
-]]
-
-local getJobIdFromKey = function (jobKey)
-  return string.match(jobKey, ".*:(.*)")
-end
-
-local getJobKeyPrefix = function (jobKey, jobId)
-  return string.sub(jobKey, 0, #jobKey - #jobId)
-end

package/dist/esm/commands/includes/getNextDelayedTimestamp.lua

@@ -1,13 +0,0 @@
---[[
-  Function to return the next delayed job timestamp.
-]] 
-local function getNextDelayedTimestamp(delayedKey)
-    local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
-    if #result then
-      local nextTimestamp = tonumber(result[2])
-      if (nextTimestamp ~= nil) then 
-        nextTimestamp = nextTimestamp / 0x1000
-      end
-      return nextTimestamp
-    end
-end

package/dist/esm/commands/includes/getTargetQueueList.lua

@@ -1,12 +0,0 @@
---[[
-  Function to check for the meta.paused key to decide if we are paused or not
-  (since an empty list and !EXISTS are not really the same).
-]]
-
-local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
-  if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
-    return waitKey
-  else
-    return pausedKey
-  end
-end

package/dist/esm/commands/includes/getTimestamp.lua

@@ -1,19 +0,0 @@
---[[
-  Function to get the latest saved timestamp.
-]]
-
-local function getTimestamp(jobKey, attributes)
-  if #attributes == 1 then
-    return rcall("HGET", jobKey, attributes[1])
-  end
-
-  local jobTs
-  for _, ts in ipairs(rcall("HMGET", jobKey, unpack(attributes))) do
-    if (ts) then
-      jobTs = ts
-      break
-    end
-  end
-  
-  return jobTs
-end

package/dist/esm/commands/includes/getZSetItems.lua

@@ -1,7 +0,0 @@
---[[
-  Function to get ZSet items.
-]]
-
-local function getZSetItems(keyName, max)
-  return rcall('ZRANGE', keyName, 0, max - 1)
-end

package/dist/esm/commands/includes/isLocked.lua

@@ -1,31 +0,0 @@
---[[
-  Function to recursively check if there are no locks
-  on the jobs to be removed.
-
-  returns:
-    boolean
-]]
-
-local function isLocked( prefix, jobId)
-  local jobKey = prefix .. jobId;
-
-  -- Check if this job is locked
-  local lockKey = jobKey .. ':lock'
-  local lock = rcall("GET", lockKey)
-  if not lock then
-      local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
-      if (#dependencies > 0) then
-          for i, childJobKey in ipairs(dependencies) do
-              -- We need to get the jobId for this job.
-              local childJobId = getJobIdFromKey(childJobKey)
-              local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
-              local result = isLocked( childJobPrefix, childJobId )
-              if result then
-                  return true
-              end
-          end
-      end
-      return false
-  end
-  return true
-end

package/dist/esm/commands/includes/moveJobFromWaitToActive.lua

@@ -1,84 +0,0 @@
-
---[[
-  Function to move job from wait state to active.
-  Input:
-    keys[1] wait key
-    keys[2] active key
-    keys[3] priority key
-    keys[4] stream events key
-    keys[5] stalled key
-
-    -- Rate limiting
-    keys[6] rate limiter key
-    keys[7] delayed key
-
-    opts - token - lock token
-    opts - lockDuration
-    opts - limiter
-]]
-
-local function moveJobFromWaitToActive(keys, keyPrefix, jobId, processedOn, opts)
-  -- Check if we need to perform rate limiting.
-  local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
-
-  if(maxJobs) then
-    local rateLimiterKey = keys[6];
-
-    local groupKey
-    local groupKeyOpt = opts['limiter'] and opts['limiter']['groupKey'] or ""
-    if groupKeyOpt ~= "" then
-      groupKey = string.match(jobId, "[^:]+$")
-      if groupKey ~= jobId then
-        rateLimiterKey = rateLimiterKey .. ":" .. groupKey
-      end
-    end
-
-    local jobCounter
-    
-    if groupKey ~= nil then
-      if rateLimiterKey ~= keys[6] then
-        jobCounter = tonumber(rcall("INCR", rateLimiterKey))
-      end
-    else
-      jobCounter = tonumber(rcall("INCR", rateLimiterKey))
-    end
-
-    local limiterDuration = opts['limiter'] and opts['limiter']['duration']
-    -- check if rate limit hit
-    if jobCounter ~= nil and jobCounter > maxJobs then
-      local exceedingJobs = jobCounter - maxJobs
-      local expireTime = tonumber(rcall("PTTL", rateLimiterKey))
-      local delay = expireTime + ((exceedingJobs - 1) * limiterDuration) / maxJobs;
-      local timestamp = delay + tonumber(processedOn)
-
-      -- put job into delayed queue
-      rcall("ZADD", keys[7], timestamp * 0x1000 + bit.band(jobCounter, 0xfff), jobId);
-      rcall("XADD", keys[4], "*", "event", "delayed", "jobId", jobId, "delay", timestamp);
-
-      -- remove from active queue
-      rcall("LREM", keys[2], 1, jobId)
-
-      -- Return when we can process more jobs
-      return expireTime
-    else
-      if jobCounter == 1 then
-        rcall("PEXPIRE", rateLimiterKey, limiterDuration)
-      end
-    end
-  end
-
-  local jobKey = keyPrefix .. jobId
-  local lockKey = jobKey .. ':lock'
-
-  -- get a lock
-  if opts['token'] ~= "0" then
-    rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
-  end
-
-  rcall("ZREM", keys[3], jobId) -- remove from priority
-  rcall("XADD", keys[4], "*", "event", "active", "jobId", jobId, "prev", "waiting")
-  rcall("HSET", jobKey, "processedOn", processedOn)
-  rcall("HINCRBY", jobKey, "attemptsMade", 1)
-
-  return {rcall("HGETALL", jobKey), jobId} -- get job data
-end

package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua

@@ -1,25 +0,0 @@
---[[
-  Function to recursively move from waitingChildren to failed.
-]]
-
-local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey, parentId, jobId, timestamp)
-  if rcall("ZREM", parentQueueKey .. ":waiting-children", parentId) == 1 then
-    rcall("ZADD", parentQueueKey .. ":failed", timestamp, parentId)
-    rcall("HMSET", parentKey, "failedReason", "child " .. jobId .. " failed", "finishedOn", timestamp)
-
-    local rawParentData = rcall("HGET", parentKey, "parent")
-
-    if rawParentData ~= false then
-      local parentData = cjson.decode(rawParentData)
-      if parentData['fpof'] then
-        moveParentFromWaitingChildrenToFailed(
-          parentData['queueKey'],
-          parentData['queueKey'] .. ':' .. parentData['id'],
-          parentData['id'],
-          parentKey,
-          timestamp
-        )
-      end
-    end
-  end
-end

package/dist/esm/commands/includes/promoteDelayedJobs.lua

@@ -1,49 +0,0 @@
---[[
-  Updates the delay set, by moving delayed jobs that should
-  be processed now to "wait".
-
-     Events:
-      'waiting'
-]]
-local rcall = redis.call
-
--- Includes
---- @include "addJobWithPriority"
---- @include "getTargetQueueList"
-
--- Try to get as much as 1000 jobs at once, and returns the nextTimestamp if
--- there are more delayed jobs to process.
-local function promoteDelayedJobs(delayedKey, waitKey, priorityKey, pausedKey,
-                                  metaKey, eventStreamKey, prefix, timestamp)
-    local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000, "LIMIT", 0, 1000)
-
-    if (#jobs > 0) then
-        rcall("ZREM", delayedKey, unpack(jobs))
-
-        -- check if we need to use push in paused instead of waiting
-        local target = getTargetQueueList(metaKey, waitKey, pausedKey)
-
-        for _, jobId in ipairs(jobs) do
-            local priority =
-                tonumber(rcall("HGET", prefix .. jobId, "priority")) or 0
-
-            if priority == 0 then
-                -- LIFO or FIFO
-                rcall("LPUSH", target, jobId)
-            else
-                addJobWithPriority(priorityKey, priority, target, jobId)
-            end
-
-            -- Emit waiting event
-            rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
-                  jobId, "prev", "delayed")
-            rcall("HSET", prefix .. jobId, "delay", 0)
-        end
-    end
-
-    local nextTimestamp = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")[2]
-    if (nextTimestamp ~= nil) then
-        nextTimestamp = nextTimestamp / 0x1000
-    end
-    return nextTimestamp
-end

package/dist/esm/commands/includes/removeJob.lua

@@ -1,13 +0,0 @@
---[[
-  Function to remove job.
-]]
-
--- Includes
---- @include "removeParentDependencyKey"
-
-local function removeJob(jobId, hard, baseKey)
-  local jobKey = baseKey .. jobId
-  removeParentDependencyKey(jobKey, hard, nil, baseKey)
-  rcall("DEL", jobKey, jobKey .. ':logs',
-    jobKey .. ':dependencies', jobKey .. ':processed')
-end

package/dist/esm/commands/includes/removeJobFromAnyState.lua

@@ -1,26 +0,0 @@
---[[
-  Function to remove from any state.
-
-  returns:
-    prev state
-]]
-
-local function removeJobFromAnyState( prefix, jobId)
-  if rcall("LREM", prefix .. "wait", 0, jobId) == 1 then
-    return "wait"
-  elseif rcall("LREM", prefix .. "paused", 0, jobId) == 1 then
-    return "paused"
-  elseif rcall("LREM", prefix .. "active", 0, jobId) == 1 then
-    return "active"
-  elseif rcall("ZREM", prefix .. "waiting-children", jobId) == 1 then
-    return "waiting-children"
-  elseif rcall("ZREM", prefix .. "delayed", jobId) == 1 then
-    return "delayed"
-  elseif rcall("ZREM", prefix .. "completed", jobId) == 1 then
-    return "completed"
-  elseif rcall("ZREM", prefix .. "failed", jobId) == 1 then
-    return "failed"
-  end
-
-  return "unknown"
-end

package/dist/esm/commands/includes/removeJobs.lua

@@ -1,38 +0,0 @@
---[[
-  Functions to remove jobs.
-]]
-
--- Includes
---- @include "batches"
-
-local function getListItems(keyName, max)
-  return rcall('LRANGE', keyName, 0, max - 1)
-end
-
---- @include "getZSetItems"
---- @include "removeJob"
-
-local function removeJobs(keys, hard, baseKey, max)
-  for i, key in ipairs(keys) do
-    removeJob(key, hard, baseKey)
-  end
-  return max - #keys
-end
-
-local function removeListJobs(keyName, hard, baseKey, max)
-  local jobs = getListItems(keyName, max)
-  local count = removeJobs(jobs, hard, baseKey, max)
-  rcall("LTRIM", keyName, #jobs, -1)
-  return count
-end
-
-local function removeZSetJobs(keyName, hard, baseKey, max)
-  local jobs = getZSetItems(keyName, max)
-  local count = removeJobs(jobs, hard, baseKey, max)
-  if(#jobs > 0) then
-    for from, to in batches(#jobs, 7000) do
-      rcall("ZREM", keyName, unpack(jobs))
-    end
-  end
-  return count
-end

package/dist/esm/commands/includes/removeJobsByMaxAge.lua

@@ -1,15 +0,0 @@
---[[
-  Functions to remove jobs by max age.
-]]
-
--- Includes
---- @include "removeJob"
-
-local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
-  local start = timestamp - maxAge * 1000
-  local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
-  for i, jobId in ipairs(jobIds) do
-    removeJob(jobId, false, prefix)
-  end
-  rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
-end

package/dist/esm/commands/includes/removeJobsByMaxCount.lua

@@ -1,15 +0,0 @@
---[[
-  Functions to remove jobs by max count.
-]]
-
--- Includes
---- @include "removeJob"
-
-local function removeJobsByMaxCount(maxCount, targetSet, prefix)
-  local start = maxCount
-  local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
-  for i, jobId in ipairs(jobIds) do
-    removeJob(jobId, false, prefix)
-  end
-  rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
-end

package/dist/esm/commands/includes/trimEvents.lua

@@ -1,12 +0,0 @@
---[[
-  Function to trim events, default 10000.
-]]
-
-local function trimEvents(metaKey, eventStreamKey)
-  local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
-  if maxEvents ~= false then
-    rcall("XTRIM", eventStreamKey, "MAXLEN", "~", maxEvents)
-  else
-    rcall("XTRIM", eventStreamKey, "MAXLEN", "~", 10000)
-  end
-end

package/dist/esm/commands/includes/updateParentDepsIfNeeded.lua

@@ -1,28 +0,0 @@
---[[
-  Validate and move or add dependencies to parent.
-]]
-
--- Includes
---- @include "addJobWithPriority"
---- @include "getTargetQueueList"
-
-local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
-  parentId, jobIdKey, returnvalue )
-  local processedSet = parentKey .. ":processed"
-  rcall("HSET", processedSet, jobIdKey, returnvalue)
-  local activeParent = rcall("ZSCORE", parentQueueKey .. ":waiting-children", parentId)
-  if rcall("SCARD", parentDependenciesKey) == 0 and activeParent then 
-    rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
-    local parentTarget = getTargetQueueList(parentQueueKey .. ":meta", parentQueueKey .. ":wait",
-      parentQueueKey .. ":paused")
-    local priority = tonumber(rcall("HGET", parentKey, "priority"))
-    -- Standard or priority add
-    if priority == 0 then
-      rcall("RPUSH", parentTarget, parentId)
-    else
-      addJobWithPriority(parentQueueKey .. ":priority", priority, parentTarget, parentId)
-    end
-
-    rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
-  end
-end

package/dist/esm/commands/isJobInList-1.lua

@@ -1,16 +0,0 @@
---[[
-  Checks if job is in a given list.
-
-  Input:
-    KEYS[1]
-    ARGV[1]
-
-  Output:
-    1 if element found in the list.
-]]
-
--- Includes
---- @include "includes/checkItemInList"
-
-local items = redis.call("LRANGE", KEYS[1] , 0, -1)
-return checkItemInList(items, ARGV[1])

package/dist/esm/commands/moveStalledJobsToWait-8.lua

@@ -1,24 +0,0 @@
---[[
-  Move stalled jobs to wait.
-
-    Input:
-      KEYS[1] 'stalled' (SET)
-      KEYS[2] 'wait',   (LIST)
-      KEYS[3] 'active', (LIST)
-      KEYS[4] 'failed', (ZSET)
-      KEYS[5] 'stalled-check', (KEY)
-      KEYS[6] 'meta', (KEY)
-      KEYS[7] 'paused', (LIST)
-      KEYS[8] 'event stream' (STREAM)
-
-      ARGV[1]  Max stalled job count
-      ARGV[2]  queue.toKey('')
-      ARGV[3]  timestamp
-      ARGV[4]  max check time
-
-    Events:
-      'stalled' with stalled job id.
-]] -- Includes
---- @include "includes/checkStalledJobs"
-return checkStalledJobs(KEYS[1], KEYS[2], KEYS[3], KEYS[4], KEYS[5], KEYS[6],
-                        KEYS[7], KEYS[8], ARGV[1], ARGV[2], ARGV[3], ARGV[4])

package/dist/esm/commands/moveToActive-9.lua

@@ -1,67 +0,0 @@
---[[
-  Move next job to be processed to active, lock it and fetch its data. The job
-  may be delayed, in that case we need to move it to the delayed set instead.
-
-  This operation guarantees that the worker owns the job during the lock
-  expiration time. The worker is responsible of keeping the lock fresh
-  so that no other worker picks this job again.
-
-  Input:
-      KEYS[1] wait key
-      KEYS[2] active key
-      KEYS[3] priority key
-      KEYS[4] stream events key
-      KEYS[5] stalled key
-
-      -- Rate limiting
-      KEYS[6] rate limiter key
-      KEYS[7] delayed key
-
-      -- Promote delayed jobs
-      KEYS[8] paused key
-      KEYS[9] meta key
-
-      -- Arguments
-      ARGV[1] key prefix
-      ARGV[2] timestamp
-      ARGV[3] optional job ID
-      ARGV[4] opts
-
-      opts - token - lock token
-      opts - lockDuration
-      opts - limiter
-]]
-local jobId
-local rcall = redis.call
-
--- Includes
---- @include "includes/moveJobFromWaitToActive"
---- @include "includes/getNextDelayedTimestamp"
---- @include "includes/promoteDelayedJobs"
-
--- Check if there are delayed jobs that we can move to wait.
-promoteDelayedJobs(KEYS[7], KEYS[1], KEYS[3], KEYS[8], KEYS[9], KEYS[4], ARGV[1], ARGV[2])
-
-if (ARGV[3] ~= "") then
-    jobId = ARGV[3]
-    -- clean stalled key
-    rcall("SREM", KEYS[5], jobId)
-else
-    -- no job ID, try non-blocking move from wait to active
-    jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
-end
-
--- If jobId is special ID 0, then there is no job to process
-if jobId == "0" then
-    rcall("LREM", KEYS[2], 1, 0)
-elseif jobId then
-    local opts = cmsgpack.unpack(ARGV[4])
-    -- this script is not really moving, it is preparing the job for processing
-    return moveJobFromWaitToActive(KEYS, ARGV[1], jobId, ARGV[2], opts)
-end
-
--- Return the timestamp for the next delayed job if any.
-local nextTimestamp = getNextDelayedTimestamp(KEYS[7])
-if (nextTimestamp ~= nil) then
-    return nextTimestamp - tonumber(ARGV[2])
-end