bullmq 2.2.0 → 2.2.1

package/dist/cjs/commands/moveToFinished-12.lua

@@ -1,201 +0,0 @@
---[[
-  Move job from active to a finished status (completed o failed)
-  A job can only be moved to completed if it was active.
-  The job must be locked before it can be moved to a finished status,
-  and the lock must be released in this script.
-
-    Input:
-      KEYS[1] wait key
-      KEYS[2] active key
-      KEYS[3] priority key
-      KEYS[4] event stream key
-      KEYS[5] stalled key
-
-      -- Rate limiting
-      KEYS[6] rate limiter key
-      KEYS[7] delayed key
-
-      KEYS[8] paused key
-
-      KEYS[9] completed/failed key
-      KEYS[10] jobId key
-      KEYS[11] meta key
-      KEYS[12] metrics key
-
-      ARGV[1]  jobId
-      ARGV[2]  timestamp
-      ARGV[3]  msg property returnvalue / failedReason
-      ARGV[4]  return value / failed reason
-      ARGV[5]  target (completed/failed)
-      ARGV[6]  event data (? maybe just send jobid).
-      ARGV[7]  fetch next?
-      ARGV[8]  keys prefix
-      ARGV[9]  opts
-
-      opts - token - lock token
-      opts - keepJobs
-      opts - lockDuration - lock duration in milliseconds
-      opts - parent - parent data
-      opts - parentKey
-      opts - attempts max attempts
-      opts - attemptsMade
-      opts - maxMetricsSize
-      opts - fpof - fail parent on fail
-
-    Output:
-      0 OK
-      -1 Missing key.
-      -2 Missing lock.
-      -3 Job not in active set
-      -4 Job has pending dependencies
-
-    Events:
-      'completed/failed'
-]]
-local rcall = redis.call
-
---- Includes
---- @include "includes/collectMetrics"
---- @include "includes/getNextDelayedTimestamp"
---- @include "includes/moveJobFromWaitToActive"
---- @include "includes/moveParentFromWaitingChildrenToFailed"
---- @include "includes/promoteDelayedJobs"
---- @include "includes/removeJobsByMaxAge"
---- @include "includes/removeJobsByMaxCount"
---- @include "includes/trimEvents"
---- @include "includes/updateParentDepsIfNeeded"
-
-local jobIdKey = KEYS[10]
-if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
-    local opts = cmsgpack.unpack(ARGV[9])
-
-    local token = opts['token']
-    local parentId = opts['parent'] and opts['parent']['id'] or ""
-    local parentQueueKey = opts['parent'] and opts['parent']['queue'] or ""
-    local parentKey = opts['parentKey'] or ""
-    local attempts = opts['attempts']
-    local attemptsMade = opts['attemptsMade']
-    local maxMetricsSize = opts['maxMetricsSize']
-    local maxCount = opts['keepJobs']['count']
-    local maxAge = opts['keepJobs']['age']
-
-    if token ~= "0" then
-        local lockKey = jobIdKey .. ':lock'
-        if rcall("GET", lockKey) == token then
-            rcall("DEL", lockKey)
-            rcall("SREM", KEYS[5], ARGV[1])
-        else
-            return -2
-        end
-    end
-
-    if rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then -- // Make sure it does not have pending dependencies
-        return -4
-    end
-
-    local jobId = ARGV[1]
-    local timestamp = ARGV[2]
-
-    -- Remove from active list (if not active we shall return error)
-    local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
-
-    if (numRemovedElements < 1) then return -3 end
-
-    -- Trim events before emiting them to avoid trimming events emitted in this script
-    trimEvents(KEYS[11], KEYS[4])
-
-    -- If job has a parent we need to
-    -- 1) remove this job id from parents dependencies
-    -- 2) move the job Id to parent "processed" set
-    -- 3) push the results into parent "results" list
-    -- 4) if parent's dependencies is empty, then move parent to "wait/paused". Note it may be a different queue!.
-    if parentId == "" and parentKey ~= "" then
-        parentId = getJobIdFromKey(parentKey)
-        parentQueueKey = getJobKeyPrefix(parentKey, ":" .. parentId)
-    end
-    if parentId ~= "" then
-        if ARGV[5] == "completed" then
-            local dependenciesSet = parentKey .. ":dependencies"
-            if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
-                updateParentDepsIfNeeded(parentKey, parentQueueKey, dependenciesSet,
-                                        parentId, jobIdKey, ARGV[4])
-            end
-        elseif opts['fpof'] then
-            moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
-        end
-    end
-
-    -- Remove job?
-    if maxCount ~= 0 then
-        local targetSet = KEYS[9]
-        -- Add to complete/failed set
-        rcall("ZADD", targetSet, timestamp, jobId)
-        rcall("HMSET", jobIdKey, ARGV[3], ARGV[4], "finishedOn", timestamp)
-        -- "returnvalue" / "failedReason" and "finishedOn"
-
-        -- Remove old jobs?
-        local prefix = ARGV[8]
-
-        if maxAge ~= nil then
-            removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
-        end
-
-        if maxCount ~= nil and maxCount > 0 then
-            removeJobsByMaxCount(maxCount, targetSet, prefix)
-        end
-    else
-        rcall("DEL", jobIdKey, jobIdKey .. ':logs', jobIdKey .. ':processed')
-    end
-
-    rcall("XADD", KEYS[4], "*", "event", ARGV[5], "jobId", jobId, ARGV[3],
-          ARGV[4])
-
-    if ARGV[5] == "failed" then
-        if tonumber(attemptsMade) >= tonumber(attempts) then
-            rcall("XADD", KEYS[4], "*", "event", "retries-exhausted", "jobId",
-                  jobId, "attemptsMade", attemptsMade)
-        end
-    end
-
-    -- Collect metrics
-    if maxMetricsSize ~= "" then
-        collectMetrics(KEYS[12], KEYS[12]..':data', maxMetricsSize, timestamp)
-    end
-
-    -- Try to get next job to avoid an extra roundtrip if the queue is not closing,
-    -- and not rate limited.
-    if (ARGV[7] == "1") then
-
-        -- Check if there are delayed jobs that can be promoted
-        promoteDelayedJobs(KEYS[7], KEYS[1], KEYS[3], KEYS[8], KEYS[11], KEYS[4], ARGV[8], timestamp)
-
-        jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
-
-        if jobId == "0" then
-            rcall("LREM", KEYS[2], 1, 0)
-        elseif jobId then
-            return moveJobFromWaitToActive(KEYS, ARGV[8], jobId, timestamp, opts)
-        end
-
-        -- Return the timestamp for the next delayed job if any.
-        local nextTimestamp = getNextDelayedTimestamp(KEYS[7])
-        if (nextTimestamp ~= nil) then
-            -- The result is guaranteed to be positive, since the
-            -- ZRANGEBYSCORE command would have return a job otherwise.
-            return nextTimestamp - timestamp
-        end
-    end
-
-    local waitLen = rcall("LLEN", KEYS[1])
-    if waitLen == 0 then
-        local activeLen = rcall("LLEN", KEYS[2])
-
-        if activeLen == 0 then
-            rcall("XADD", KEYS[4], "*", "event", "drained")
-        end
-    end  
-
-    return 0
-else
-    return -1
-end

package/dist/cjs/commands/obliterate-2.lua

@@ -1,94 +0,0 @@
---[[
-  Completely obliterates a queue and all of its contents
-  
-  Input:
-    KEYS[1] meta
-    KEYS[2] base
-
-    ARGV[1] count
-    ARGV[2] force
-]]
-
--- This command completely destroys a queue including all of its jobs, current or past 
--- leaving no trace of its existence. Since this script needs to iterate to find all the job
--- keys, consider that this call may be slow for very large queues.
-
--- The queue needs to be "paused" or it will return an error
--- If the queue has currently active jobs then the script by default will return error,
--- however this behaviour can be overrided using the 'force' option.
-local maxCount = tonumber(ARGV[1])
-local baseKey = KEYS[2]
-
-local rcall = redis.call
-
---- @include "includes/removeJobs"
-
-local function removeLockKeys(keys)
-  for i, key in ipairs(keys) do
-    rcall("DEL", baseKey .. key .. ':lock')
-  end
-end
-
--- 1) Check if paused, if not return with error.
-if rcall("HEXISTS", KEYS[1], "paused") ~= 1 then
-  return -1 -- Error, NotPaused
-end
-
--- 2) Check if there are active jobs, if there are and not "force" return error.
-local activeKey = baseKey .. 'active'
-local activeJobs = getListItems(activeKey, maxCount)
-if (#activeJobs > 0) then
-  if(ARGV[2] == "") then 
-    return -2 -- Error, ExistActiveJobs
-  end
-end
-
-removeLockKeys(activeJobs)
-maxCount = removeJobs(activeJobs, true, baseKey, maxCount)
-rcall("LTRIM", activeKey, #activeJobs, -1)
-if(maxCount <= 0) then
-  return 1
-end
-
-local delayedKey = baseKey .. 'delayed'
-maxCount = removeZSetJobs(delayedKey, true, baseKey, maxCount)
-if(maxCount <= 0) then
-  return 1
-end
-
-local completedKey = baseKey .. 'completed'
-maxCount = removeZSetJobs(completedKey, true, baseKey, maxCount)
-if(maxCount <= 0) then
-  return 1
-end
-
-local waitKey = baseKey .. 'paused'
-maxCount = removeListJobs(waitKey, true, baseKey, maxCount)
-if(maxCount <= 0) then
-  return 1
-end
-
-local failedKey = baseKey .. 'failed'
-maxCount = removeZSetJobs(failedKey, true, baseKey, maxCount)
-if(maxCount <= 0) then
-  return 1
-end
-
-if(maxCount > 0) then
-  rcall("DEL", 
-    baseKey .. 'priority',
-    baseKey .. 'events',
-    baseKey .. 'delay', 
-    baseKey .. 'stalled-check',
-    baseKey .. 'stalled',
-    baseKey .. 'id',
-    baseKey .. 'meta',
-    baseKey .. 'repeat',
-    baseKey .. 'metrics:completed',
-    baseKey .. 'metrics:completed:data',
-    baseKey .. 'metrics:failed',
-    baseKey .. 'metrics:failed:data')
-  return 0
-else
-  return 1
-end

package/dist/cjs/commands/removeJob-1.lua

@@ -1,72 +0,0 @@
---[[
-    Remove a job from all the queues it may be in as well as all its data.
-    In order to be able to remove a job, it cannot be active.
-
-    Input:
-      KEYS[1] queue prefix
-      ARGV[1] jobId
-
-    Events:
-      'removed'
-]]
-
-local rcall = redis.call
-
--- Includes
---- @include "includes/destructureJobKey"
---- @include "includes/isLocked"
---- @include "includes/removeJobFromAnyState"
---- @include "includes/removeParentDependencyKey"
-
-local function removeJob( prefix, jobId, parentKey)
-    local jobKey = prefix .. jobId;
-
-    removeParentDependencyKey(jobKey, false, parentKey)
-
-    -- Check if this job has children
-    -- If so, we are going to try to remove the children recursively in deep first way because
-    -- if some job is locked we must exit with and error.
-    --local countProcessed = rcall("HLEN", jobKey .. ":processed")
-    local processed = rcall("HGETALL", jobKey .. ":processed")
-
-    if (#processed > 0) then
-        for i = 1, #processed, 2 do
-            local childJobId = getJobIdFromKey(processed[i])
-            local childJobPrefix = getJobKeyPrefix(processed[i], childJobId)
-            removeJob( childJobPrefix, childJobId, jobKey )
-        end
-    end
-
-    local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
-    if (#dependencies > 0) then
-        for i, childJobKey in ipairs(dependencies) do
-            -- We need to get the jobId for this job.
-            local childJobId = getJobIdFromKey(childJobKey)
-            local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
-            removeJob( childJobPrefix, childJobId, jobKey )
-        end
-    end
-
-    local prev = removeJobFromAnyState(prefix, jobId)
-
-    rcall("ZREM", prefix .. "priority", jobId)
-    rcall("DEL", jobKey, jobKey .. ":logs", jobKey .. ":dependencies", jobKey .. ":processed")
-
-    -- -- delete keys related to rate limiter
-        -- local limiterIndexTable = KEYS[10] .. ":index"
-        -- local limitedSetKey = rcall("HGET", limiterIndexTable, jobId)
-        -- if limitedSetKey then
-        --     rcall("SREM", limitedSetKey, jobId)
-        --     rcall("HDEL", limiterIndexTable, jobId)
-    -- end
-
-    rcall("XADD", prefix .. "events", "*", "event", "removed", "jobId", jobId, "prev", prev);
-end
-
-local prefix = KEYS[1]
-
-if not isLocked(prefix, ARGV[1]) then
-    removeJob(prefix, ARGV[1])
-    return 1
-end
-return 0

package/dist/cjs/commands/updateData-1.lua

@@ -1,16 +0,0 @@
---[[
-  Update job data
-
-  Input:
-    KEYS[1] Job id key
-  
-    ARGV[1] data        
-]]
-local rcall = redis.call
-
-if rcall("EXISTS",KEYS[1]) == 1 then -- // Make sure job exists
-  rcall("HSET", KEYS[1], "data", ARGV[1])
-  return 0
-else
-  return -1
-end

package/dist/esm/commands/cleanJobsInSet-2.lua

@@ -1,46 +0,0 @@
---[[
-  Remove jobs from the specific set.
-
-  Input:
-    KEYS[1]  set key,
-    KEYS[2]  events stream key
-
-    ARGV[1]  jobKey prefix
-    ARGV[2]  timestamp
-    ARGV[3]  limit the number of jobs to be removed. 0 is unlimited
-    ARGV[4]  set name, can be any of 'wait', 'active', 'paused', 'delayed', 'completed', or 'failed'
-]]
-local rcall = redis.call
-local rangeStart = 0
-local rangeEnd = -1
-
-local limit = tonumber(ARGV[3])
-
--- If we're only deleting _n_ items, avoid retrieving all items
--- for faster performance
---
--- Start from the tail of the list, since that's where oldest elements
--- are generally added for FIFO lists
-if limit > 0 then
-  rangeStart = -1 - limit + 1
-  rangeEnd = -1
-end
-
--- Includes
---- @include "includes/cleanList"
---- @include "includes/cleanSet"
-
-local result
-if ARGV[4] == "active" then
-  result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], false)
-elseif ARGV[4] == "delayed" then
-  result = cleanSet(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], limit, {"processedOn", "timestamp"})
-elseif ARGV[4] == "wait" or ARGV[4] == "paused" then
-  result = cleanList(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], true)
-else
-  result = cleanSet(KEYS[1], ARGV[1], rangeStart, rangeEnd, ARGV[2], limit, {"finishedOn"} )
-end
-
-rcall("XADD", KEYS[2], "*", "event", "cleaned", "count", result[2])
-
-return result[1]

package/dist/esm/commands/drain-4.lua

@@ -1,25 +0,0 @@
---[[
-  Drains the queue, removes all jobs that are waiting
-  or delayed, but not active, completed or failed
-
-  Input:
-    KEYS[1] 'wait',
-    KEYS[2] 'paused'
-    KEYS[3] 'delayed'
-    KEYS[4] 'priority'
-
-    ARGV[1]  queue key prefix
-]]
-local rcall = redis.call
-local queueBaseKey = ARGV[1]
-
---- @include "includes/removeJobs"
-
-removeListJobs(KEYS[1], true, queueBaseKey, 0) --wait
-removeListJobs(KEYS[2], true, queueBaseKey, 0) --paused
-
-if KEYS[3] ~= "" then
-  removeZSetJobs(KEYS[3], true, queueBaseKey, 0) --delayed
-end
-
-rcall("DEL", KEYS[4])

package/dist/esm/commands/includes/addJobWithPriority.lua

@@ -1,16 +0,0 @@
---[[
-  Function to add job considering priority.
-]]
-
-local function addJobWithPriority(priorityKey, priority, targetKey, jobId)
-  rcall("ZADD", priorityKey, priority, jobId)
-  local count = rcall("ZCOUNT", priorityKey, 0, priority)
-
-  local len = rcall("LLEN", targetKey)
-  local id = rcall("LINDEX", targetKey, len - (count - 1))
-  if id then
-    rcall("LINSERT", targetKey, "BEFORE", id, jobId)
-  else
-    rcall("RPUSH", targetKey, jobId)
-  end
-end

package/dist/esm/commands/includes/batches.lua

@@ -1,18 +0,0 @@
---[[
-  Function to loop in batches.
-  Just a bit of warning, some commands as ZREM
-  could receive a maximum of 7000 parameters per call.
-]]
-
-local function batches(n, batchSize)
-  local i = 0
-
-  return function()
-    local from = i * batchSize + 1
-    i = i + 1
-    if (from <= n) then
-      local to = math.min(from + batchSize - 1, n)
-      return from, to
-    end
-  end
-end

package/dist/esm/commands/includes/checkItemInList.lua

@@ -1,12 +0,0 @@
---[[
-  Functions to check if a item belongs to a list.
-]]
-
-local function checkItemInList(list, item)
-  for _, v in pairs(list) do
-    if v == item then
-      return 1
-    end
-  end
-  return nil
-end

package/dist/esm/commands/includes/checkStalledJobs.lua

@@ -1,137 +0,0 @@
---[[
-  Move stalled jobs to wait.
-
-    Input:
-      stalledKey 'stalled' (SET)
-      waitKey 'wait',   (LIST)
-      activeKey 'active', (LIST)
-      failedKey 'failed', (ZSET)
-      stalledCheckKey 'stalled-check', (KEY)
-      metaKey 'meta', (KEY)
-      pausedKey 'paused', (LIST)
-      eventStreamKey 'event stream' (STREAM)
-
-      maxStalledJobCount  Max stalled job count
-      queueKeyPrefix  queue.toKey('')
-      timestamp  timestamp
-      maxCheckTime  max check time
-
-    Events:
-      'stalled' with stalled job id.
-]] local rcall = redis.call
-
--- Includes
---- @include "batches"
---- @include "getTargetQueueList"
---- @include "removeJob"
---- @include "removeJobsByMaxAge"
---- @include "removeJobsByMaxCount"
---- @include "trimEvents"
-
--- Check if we need to check for stalled jobs now.
-
-local function checkStalledJobs(stalledKey, waitKey, activeKey, failedKey, stalledCheckKey,
-    metaKey, pausedKey, eventStreamKey, maxStalledJobCount, queueKeyPrefix, timestamp, maxCheckTime)
-    if rcall("EXISTS", stalledCheckKey) == 1 then 
-        return {{}, {}}
-    end
-
-    rcall("SET", stalledCheckKey, timestamp, "PX", maxCheckTime)
-
-    -- Trim events before emiting them to avoid trimming events emitted in this script
-    trimEvents(metaKey, eventStreamKey)
-
-    -- Move all stalled jobs to wait
-    local stalling = rcall('SMEMBERS', stalledKey)
-    local stalled = {}
-    local failed = {}
-    if (#stalling > 0) then
-        rcall('DEL', stalledKey)
-
-        local MAX_STALLED_JOB_COUNT = tonumber(maxStalledJobCount)
-
-        -- Remove from active list
-        for i, jobId in ipairs(stalling) do
-
-            if jobId == '0' then
-                -- If the jobId is a delay marker ID we just remove it.
-                local removed = rcall("LREM", activeKey, 1, jobId)
-            else
-                local jobKey = queueKeyPrefix .. jobId
-
-                -- Check that the lock is also missing, then we can handle this job as really stalled.
-                if (rcall("EXISTS", jobKey .. ":lock") == 0) then
-                    --  Remove from the active queue.
-                    local removed = rcall("LREM", activeKey, 1, jobId)
-
-                    if (removed > 0) then
-                        -- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
-                        local stalledCount =
-                            rcall("HINCRBY", jobKey, "stalledCounter", 1)
-                        if (stalledCount > MAX_STALLED_JOB_COUNT) then
-                            local rawOpts = rcall("HGET", jobKey, "opts")
-                            local opts = cjson.decode(rawOpts)
-                            local removeOnFailType = type(opts["removeOnFail"])
-                            rcall("ZADD", failedKey, timestamp, jobId)
-                            local failedReason =
-                                "job stalled more than allowable limit"
-                            rcall("HMSET", jobKey, "failedReason", failedReason,
-                                "finishedOn", timestamp)
-                            rcall("XADD", eventStreamKey, "*", "event", "failed", "jobId",
-                                jobId, 'prev', 'active', 'failedReason',
-                                failedReason)
-
-                            if removeOnFailType == "number" then
-                                removeJobsByMaxCount(opts["removeOnFail"], failedKey,
-                                                    queueKeyPrefix)
-                            elseif removeOnFailType == "boolean" then
-                                if opts["removeOnFail"] then
-                                    removeJob(jobId, false, queueKeyPrefix)
-                                    rcall("ZREM", failedKey, jobId)
-                                end
-                            elseif removeOnFailType ~= "nil" then
-                                local maxAge = opts["removeOnFail"]["age"]
-                                local maxCount = opts["removeOnFail"]["count"]
-
-                                if maxAge ~= nil then
-                                    removeJobsByMaxAge(timestamp, maxAge, failedKey,
-                                                    queueKeyPrefix)
-                                end
-
-                                if maxCount ~= nil and maxCount > 0 then
-                                    removeJobsByMaxCount(maxCount, failedKey, queueKeyPrefix)
-                                end
-                            end
-
-                            table.insert(failed, jobId)
-                        else
-                            local target = getTargetQueueList(metaKey, waitKey,
-                                                            pausedKey)
-
-                            -- Move the job back to the wait queue, to immediately be picked up by a waiting worker.
-                            rcall("RPUSH", target, jobId)
-                            rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
-                                jobId, 'prev', 'active')
-
-                            -- Emit the stalled event
-                            rcall("XADD", eventStreamKey, "*", "event", "stalled", "jobId",
-                                jobId)
-                            table.insert(stalled, jobId)
-                        end
-                    end
-                end
-            end
-        end
-    end
-
-    -- Mark potentially stalled jobs
-    local active = rcall('LRANGE', activeKey, 0, -1)
-
-    if (#active > 0) then
-        for from, to in batches(#active, 7000) do
-            rcall('SADD', stalledKey, unpack(active, from, to))
-        end
-    end
-
-    return {failed, stalled}
-end