bullmq 4.0.0-beta.1 → 4.0.0

package/dist/cjs/commands/moveToFinished-13.lua

@@ -0,0 +1,250 @@
+--[[
+  Move job from active to a finished status (completed o failed)
+  A job can only be moved to completed if it was active.
+  The job must be locked before it can be moved to a finished status,
+  and the lock must be released in this script.
+
+    Input:
+      KEYS[1] wait key
+      KEYS[2] active key
+      KEYS[3] prioritized key
+      KEYS[4] event stream key
+      KEYS[5] stalled key
+
+      -- Rate limiting
+      KEYS[6] rate limiter key
+      KEYS[7] delayed key
+
+      KEYS[8] paused key
+      KEYS[9] meta key
+      KEYS[10] pc priority counter
+
+      KEYS[11] completed/failed key
+      KEYS[12] jobId key
+      KEYS[13] metrics key
+
+      ARGV[1]  jobId
+      ARGV[2]  timestamp
+      ARGV[3]  msg property returnvalue / failedReason
+      ARGV[4]  return value / failed reason
+      ARGV[5]  target (completed/failed)
+      ARGV[6]  event data (? maybe just send jobid).
+      ARGV[7]  fetch next?
+      ARGV[8]  keys prefix
+      ARGV[9]  opts
+
+      opts - token - lock token
+      opts - keepJobs
+      opts - lockDuration - lock duration in milliseconds
+      opts - attempts max attempts
+      opts - attemptsMade
+      opts - maxMetricsSize
+      opts - fpof - fail parent on fail
+
+    Output:
+      0 OK
+      -1 Missing key.
+      -2 Missing lock.
+      -3 Job not in active set
+      -4 Job has pending dependencies
+      -6 Lock is not owned by this client
+
+    Events:
+      'completed/failed'
+]]
+local rcall = redis.call
+
+--- Includes
+--- @include "includes/collectMetrics"
+--- @include "includes/getNextDelayedTimestamp"
+--- @include "includes/moveJobFromPriorityToActive"
+--- @include "includes/prepareJobForProcessing"
+--- @include "includes/moveParentFromWaitingChildrenToFailed"
+--- @include "includes/promoteDelayedJobs"
+--- @include "includes/removeJobsByMaxAge"
+--- @include "includes/removeJobsByMaxCount"
+--- @include "includes/removeParentDependencyKey"
+--- @include "includes/trimEvents"
+--- @include "includes/updateParentDepsIfNeeded"
+--- @include "includes/getRateLimitTTL"
+
+local jobIdKey = KEYS[12]
+if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
+    local opts = cmsgpack.unpack(ARGV[9])
+
+    local token = opts['token']
+    local attempts = opts['attempts']
+    local attemptsMade = opts['attemptsMade']
+    local maxMetricsSize = opts['maxMetricsSize']
+    local maxCount = opts['keepJobs']['count']
+    local maxAge = opts['keepJobs']['age']
+
+    if token ~= "0" then
+        local lockKey = jobIdKey .. ':lock'
+        local lockToken = rcall("GET", lockKey)
+        if lockToken == token then
+            rcall("DEL", lockKey)
+            rcall("SREM", KEYS[5], ARGV[1])
+        else
+            if lockToken then
+                -- Lock exists but token does not match
+                return -6
+            else
+                -- Lock is missing completely
+                return -2
+            end
+        end
+    end
+
+    if rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then -- // Make sure it does not have pending dependencies
+        return -4
+    end
+
+    local parentReferences = rcall("HMGET", jobIdKey, "parentKey", "parent")
+    local parentKey = parentReferences[1] or ""
+    local parentId = ""
+    local parentQueueKey = ""
+    if parentReferences[2] ~= false then
+        local jsonDecodedParent = cjson.decode(parentReferences[2])
+        parentId = jsonDecodedParent['id']
+        parentQueueKey = jsonDecodedParent['queueKey']
+    end
+
+    local jobId = ARGV[1]
+    local timestamp = ARGV[2]
+
+    -- Remove from active list (if not active we shall return error)
+    local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
+
+    if (numRemovedElements < 1) then return -3 end
+
+    -- Trim events before emiting them to avoid trimming events emitted in this script
+    trimEvents(KEYS[9], KEYS[4])
+
+    -- If job has a parent we need to
+    -- 1) remove this job id from parents dependencies
+    -- 2) move the job Id to parent "processed" set
+    -- 3) push the results into parent "results" list
+    -- 4) if parent's dependencies is empty, then move parent to "wait/paused". Note it may be a different queue!.
+    if parentId == "" and parentKey ~= "" then
+        parentId = getJobIdFromKey(parentKey)
+        parentQueueKey = getJobKeyPrefix(parentKey, ":" .. parentId)
+    end
+    if parentId ~= "" then
+        if ARGV[5] == "completed" then
+            local dependenciesSet = parentKey .. ":dependencies"
+            if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
+                updateParentDepsIfNeeded(parentKey, parentQueueKey,
+                                         dependenciesSet, parentId, jobIdKey,
+                                         ARGV[4], timestamp)
+            end
+        elseif opts['fpof'] then
+            moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey,
+                                                  parentId, jobIdKey, timestamp)
+        end
+    end
+
+    -- Remove job?
+    if maxCount ~= 0 then
+        local targetSet = KEYS[11]
+        -- Add to complete/failed set
+        rcall("ZADD", targetSet, timestamp, jobId)
+        rcall("HMSET", jobIdKey, ARGV[3], ARGV[4], "finishedOn", timestamp)
+        -- "returnvalue" / "failedReason" and "finishedOn"
+
+        -- Remove old jobs?
+        local prefix = ARGV[8]
+
+        if maxAge ~= nil then
+            removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
+        end
+
+        if maxCount ~= nil and maxCount > 0 then
+            removeJobsByMaxCount(maxCount, targetSet, prefix)
+        end
+    else
+        rcall("DEL", jobIdKey, jobIdKey .. ':logs', jobIdKey .. ':processed')
+        if parentKey ~= "" then
+            removeParentDependencyKey(jobIdKey, false, parentKey)
+        end
+    end
+
+    rcall("XADD", KEYS[4], "*", "event", ARGV[5], "jobId", jobId, ARGV[3],
+          ARGV[4])
+
+    if ARGV[5] == "failed" then
+        if tonumber(attemptsMade) >= tonumber(attempts) then
+            rcall("XADD", KEYS[4], "*", "event", "retries-exhausted", "jobId",
+                  jobId, "attemptsMade", attemptsMade)
+        end
+    end
+
+    -- Collect metrics
+    if maxMetricsSize ~= "" then
+        collectMetrics(KEYS[13], KEYS[13] .. ':data', maxMetricsSize, timestamp)
+    end
+
+    -- Try to get next job to avoid an extra roundtrip if the queue is not closing,
+    -- and not rate limited.
+    if (ARGV[7] == "1") then
+
+        local target, paused = getTargetQueueList(KEYS[9], KEYS[1], KEYS[8])
+
+        -- Check if there are delayed jobs that can be promoted
+        promoteDelayedJobs(KEYS[7], KEYS[1], target, KEYS[3],
+                           KEYS[4], ARGV[8], timestamp, paused, KEYS[10])
+
+        local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
+        -- Check if we are rate limited first.
+        local expireTime = getRateLimitTTL(maxJobs, KEYS[6])
+
+        if expireTime > 0 then return {0, 0, expireTime, 0} end
+
+        -- paused queue
+        if paused then return {0, 0, 0, 0} end
+
+        jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
+
+        -- If jobId is special ID 0:delay, then there is no job to process
+        if jobId then
+            if string.sub(jobId, 1, 2) == "0:" then
+                rcall("LREM", KEYS[2], 1, jobId)
+            else
+                -- this script is not really moving, it is preparing the job for processing
+                return prepareJobForProcessing(KEYS, ARGV[8], target, jobId, timestamp, maxJobs,
+                    expireTime, paused, opts)
+            end
+        else
+            jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
+            if jobId then
+                return prepareJobForProcessing(KEYS, ARGV[8], target, jobId, timestamp, maxJobs,
+                    expireTime, paused, opts)
+            end
+        end
+
+        -- Return the timestamp for the next delayed job if any.
+        local nextTimestamp = getNextDelayedTimestamp(KEYS[7])
+        if nextTimestamp ~= nil then
+            -- The result is guaranteed to be positive, since the
+            -- ZRANGEBYSCORE command would have return a job otherwise.
+            return {0, 0, 0, nextTimestamp}
+        end
+    end
+
+    local waitLen = rcall("LLEN", KEYS[1])
+    if waitLen == 0 then
+        local activeLen = rcall("LLEN", KEYS[2])
+
+        if activeLen == 0 then
+            local prioritizedLen = rcall("ZCARD", KEYS[3])
+
+            if prioritizedLen == 0 then
+                rcall("XADD", KEYS[4], "*", "event", "drained")
+            end
+        end
+    end
+
+    return 0
+else
+    return -1
+end

package/dist/cjs/commands/moveToWaitingChildren-4.lua

@@ -0,0 +1,62 @@
+--[[
+  Moves job from active to waiting children set.
+
+  Input:
+    KEYS[1] lock key
+    KEYS[2] active key
+    KEYS[3] waitChildrenKey key
+    KEYS[4] job key
+
+    ARGV[1] token
+    ARGV[2] child key
+    ARGV[3] timestamp
+    ARGV[4] the id of the job
+    
+  Output:
+    0 - OK
+    1 - There are not pending dependencies.
+   -1 - Missing job.
+   -2 - Missing lock
+   -3 - Job not in active set
+]]
+local rcall = redis.call
+
+local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId, timestamp, lockKey, token)
+  if token ~= "0" then
+    if rcall("GET", lockKey) == token then
+      rcall("DEL", lockKey)
+    else
+      return -2
+    end
+  end
+
+  local score = tonumber(timestamp)
+
+  local numRemovedElements = rcall("LREM", activeKey, -1, jobId)
+
+  if(numRemovedElements < 1) then
+    return -3
+  end
+
+  rcall("ZADD", waitingChildrenKey, score, jobId)
+
+  return 0
+end
+
+if rcall("EXISTS", KEYS[4]) == 1 then
+  if ARGV[2] ~= "" then
+    if rcall("SISMEMBER", KEYS[4] .. ":dependencies", ARGV[2]) ~= 0 then
+      return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], ARGV[1])
+    end
+
+    return 1
+  else
+    if rcall("SCARD", KEYS[4] .. ":dependencies") ~= 0 then 
+      return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], ARGV[1])
+    end
+
+    return 1
+  end
+end
+
+return -1

package/dist/cjs/commands/obliterate-2.lua

@@ -0,0 +1,102 @@
+--[[
+  Completely obliterates a queue and all of its contents
+  
+  Input:
+    KEYS[1] meta
+    KEYS[2] base
+
+    ARGV[1] count
+    ARGV[2] force
+]]
+
+-- This command completely destroys a queue including all of its jobs, current or past 
+-- leaving no trace of its existence. Since this script needs to iterate to find all the job
+-- keys, consider that this call may be slow for very large queues.
+
+-- The queue needs to be "paused" or it will return an error
+-- If the queue has currently active jobs then the script by default will return error,
+-- however this behaviour can be overrided using the 'force' option.
+local maxCount = tonumber(ARGV[1])
+local baseKey = KEYS[2]
+
+local rcall = redis.call
+
+-- Includes
+--- @include "includes/removeJobs"
+--- @include "includes/removeZSetJobs"
+
+local function removeLockKeys(keys)
+  for i, key in ipairs(keys) do
+    rcall("DEL", baseKey .. key .. ':lock')
+  end
+end
+
+-- 1) Check if paused, if not return with error.
+if rcall("HEXISTS", KEYS[1], "paused") ~= 1 then
+  return -1 -- Error, NotPaused
+end
+
+-- 2) Check if there are active jobs, if there are and not "force" return error.
+local activeKey = baseKey .. 'active'
+local activeJobs = getListItems(activeKey, maxCount)
+if (#activeJobs > 0) then
+  if(ARGV[2] == "") then 
+    return -2 -- Error, ExistActiveJobs
+  end
+end
+
+removeLockKeys(activeJobs)
+maxCount = removeJobs(activeJobs, true, baseKey, maxCount)
+rcall("LTRIM", activeKey, #activeJobs, -1)
+if(maxCount <= 0) then
+  return 1
+end
+
+local delayedKey = baseKey .. 'delayed'
+maxCount = removeZSetJobs(delayedKey, true, baseKey, maxCount)
+if(maxCount <= 0) then
+  return 1
+end
+
+local completedKey = baseKey .. 'completed'
+maxCount = removeZSetJobs(completedKey, true, baseKey, maxCount)
+if(maxCount <= 0) then
+  return 1
+end
+
+local waitKey = baseKey .. 'paused'
+maxCount = removeListJobs(waitKey, true, baseKey, maxCount)
+if(maxCount <= 0) then
+  return 1
+end
+
+local prioritizedKey = baseKey .. 'prioritized'
+maxCount = removeZSetJobs(prioritizedKey, true, baseKey, maxCount)
+if(maxCount <= 0) then
+  return 1
+end
+
+local failedKey = baseKey .. 'failed'
+maxCount = removeZSetJobs(failedKey, true, baseKey, maxCount)
+if(maxCount <= 0) then
+  return 1
+end
+
+if(maxCount > 0) then
+  rcall("DEL",
+    baseKey .. 'events',
+    baseKey .. 'delay', 
+    baseKey .. 'stalled-check',
+    baseKey .. 'stalled',
+    baseKey .. 'id',
+    baseKey .. 'pc',
+    baseKey .. 'meta',
+    baseKey .. 'repeat',
+    baseKey .. 'metrics:completed',
+    baseKey .. 'metrics:completed:data',
+    baseKey .. 'metrics:failed',
+    baseKey .. 'metrics:failed:data')
+  return 0
+else
+  return 1
+end

package/dist/cjs/commands/pause-5.lua

@@ -0,0 +1,36 @@
+--[[
+  Pauses or resumes a queue globably.
+
+  Input:
+    KEYS[1] 'wait' or 'paused''
+    KEYS[2] 'paused' or 'wait'
+    KEYS[3] 'meta'
+    KEYS[4] 'prioritized'
+    KEYS[5] events stream key
+
+    ARGV[1] 'paused' or 'resumed'
+
+  Event:
+    publish paused or resumed event.
+]]
+local rcall = redis.call
+
+-- Includes
+--- @include "includes/addPriorityMarkerIfNeeded"
+
+if rcall("EXISTS", KEYS[1]) == 1 then
+  rcall("RENAME", KEYS[1], KEYS[2])
+end
+
+if ARGV[1] == "paused" then
+  rcall("HSET", KEYS[3], "paused", 1)
+else
+  rcall("HDEL", KEYS[3], "paused")
+  local priorityCount = rcall("ZCARD", KEYS[4])
+
+  if priorityCount > 0 then
+    addPriorityMarkerIfNeeded(KEYS[2])
+  end
+end
+
+rcall("XADD", KEYS[5], "*", "event", ARGV[1]);

package/dist/cjs/commands/promote-7.lua

@@ -0,0 +1,57 @@
+--[[
+  Promotes a job that is currently "delayed" to the "waiting" state
+
+    Input:
+      KEYS[1] 'delayed'
+      KEYS[2] 'wait'
+      KEYS[3] 'paused'
+      KEYS[4] 'meta'
+      KEYS[5] 'prioritized'
+      KEYS[6] 'pc' priority counter
+      KEYS[7] 'event stream'
+
+      ARGV[1]  queue.toKey('')
+      ARGV[2]  jobId
+
+    Output:
+       0 - OK
+      -3 - Job not in delayed zset.
+
+    Events:
+      'waiting'
+]]
+local rcall = redis.call
+local jobId = ARGV[2]
+
+-- Includes
+--- @include "includes/addJobWithPriority"
+--- @include "includes/getTargetQueueList"
+
+if rcall("ZREM", KEYS[1], jobId) == 1 then
+  local jobKey = ARGV[1] .. jobId
+  local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
+  local target, paused = getTargetQueueList(KEYS[4], KEYS[2], KEYS[3])
+
+  -- Remove delayed "marker" from the wait list if there is any.
+  -- Since we are adding a job we do not need the marker anymore.
+  local marker = rcall("LINDEX", target, 0)
+  if marker and string.sub(marker, 1, 2) == "0:" then
+    rcall("LPOP", target)
+  end
+
+  if priority == 0 then
+    -- LIFO or FIFO
+    rcall("LPUSH", target, jobId)
+  else
+    addJobWithPriority(KEYS[2], KEYS[5], priority, paused, jobId, KEYS[6])
+  end
+
+  -- Emit waiting event (wait..ing@token)
+  rcall("XADD", KEYS[7], "*", "event", "waiting", "jobId", jobId, "prev", "delayed");
+
+  rcall("HSET", jobKey, "delay", 0)
+
+  return 0
+else
+  return -3
+end

package/dist/cjs/commands/releaseLock-1.lua

@@ -0,0 +1,19 @@
+--[[
+  Release lock
+
+    Input:
+      KEYS[1] 'lock',
+    
+      ARGV[1]  token
+      ARGV[2]  lock duration in milliseconds
+      
+    Output:
+      "OK" if lock extented succesfully.
+]]
+local rcall = redis.call
+
+if rcall("GET", KEYS[1]) == ARGV[1] then
+  return rcall("DEL", KEYS[1])
+else
+  return 0
+end

package/dist/cjs/commands/removeJob-1.lua

@@ -0,0 +1,71 @@
+--[[
+    Remove a job from all the queues it may be in as well as all its data.
+    In order to be able to remove a job, it cannot be active.
+
+    Input:
+      KEYS[1] queue prefix
+      ARGV[1] jobId
+
+    Events:
+      'removed'
+]]
+
+local rcall = redis.call
+
+-- Includes
+--- @include "includes/destructureJobKey"
+--- @include "includes/isLocked"
+--- @include "includes/removeJobFromAnyState"
+--- @include "includes/removeParentDependencyKey"
+
+local function removeJob( prefix, jobId, parentKey)
+    local jobKey = prefix .. jobId;
+
+    removeParentDependencyKey(jobKey, false, parentKey)
+
+    -- Check if this job has children
+    -- If so, we are going to try to remove the children recursively in deep first way because
+    -- if some job is locked we must exit with and error.
+    --local countProcessed = rcall("HLEN", jobKey .. ":processed")
+    local processed = rcall("HGETALL", jobKey .. ":processed")
+
+    if (#processed > 0) then
+        for i = 1, #processed, 2 do
+            local childJobId = getJobIdFromKey(processed[i])
+            local childJobPrefix = getJobKeyPrefix(processed[i], childJobId)
+            removeJob( childJobPrefix, childJobId, jobKey )
+        end
+    end
+
+    local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
+    if (#dependencies > 0) then
+        for i, childJobKey in ipairs(dependencies) do
+            -- We need to get the jobId for this job.
+            local childJobId = getJobIdFromKey(childJobKey)
+            local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
+            removeJob( childJobPrefix, childJobId, jobKey )
+        end
+    end
+
+    local prev = removeJobFromAnyState(prefix, jobId)
+
+    rcall("DEL", jobKey, jobKey .. ":logs", jobKey .. ":dependencies", jobKey .. ":processed")
+
+    -- -- delete keys related to rate limiter
+        -- local limiterIndexTable = KEYS[10] .. ":index"
+        -- local limitedSetKey = rcall("HGET", limiterIndexTable, jobId)
+        -- if limitedSetKey then
+        --     rcall("SREM", limitedSetKey, jobId)
+        --     rcall("HDEL", limiterIndexTable, jobId)
+    -- end
+
+    rcall("XADD", prefix .. "events", "*", "event", "removed", "jobId", jobId, "prev", prev);
+end
+
+local prefix = KEYS[1]
+
+if not isLocked(prefix, ARGV[1]) then
+    removeJob(prefix, ARGV[1])
+    return 1
+end
+return 0

package/{src → dist/cjs}/commands/removeRepeatable-2.lua

@@ -8,6 +8,13 @@
     ARGV[1] repeat job id
     ARGV[2] repeat job key
     ARGV[3] queue key
+
+  Output:
+    0 - OK
+    1 - Missing repeat job
+
+  Events:
+    'removed'
 ]]
 local rcall = redis.call
 local millis = rcall("ZSCORE", KEYS[1], ARGV[2])
@@ -17,7 +24,12 @@ if(millis) then
   local repeatJobId = ARGV[1] .. millis
   if(rcall("ZREM", KEYS[2], repeatJobId) == 1) then
     rcall("DEL", ARGV[3] .. repeatJobId)
+    rcall("XADD", ARGV[3] .. "events", "*", "event", "removed", "jobId", repeatJobId, "prev", "delayed");
   end
 end
 
-rcall("ZREM", KEYS[1], ARGV[2]);
+if(rcall("ZREM", KEYS[1], ARGV[2]) == 1) then
+  return 0
+end
+
+return 1

package/dist/cjs/commands/reprocessJob-6.lua

@@ -0,0 +1,43 @@
+--[[
+  Attempts to reprocess a job
+
+  Input:
+    KEYS[1] job key
+    KEYS[2] events stream
+    KEYS[3] job state
+    KEYS[4] wait key
+    KEYS[5] meta
+    KEYS[6] paused key
+
+    ARGV[1] job.id
+    ARGV[2] (job.opts.lifo ? 'R' : 'L') + 'PUSH'
+    ARGV[3] propVal - failedReason/returnvalue
+    ARGV[4] prev state - failed/completed
+
+  Output:
+     1 means the operation was a success
+    -1 means the job does not exist
+    -3 means the job was not found in the expected set.
+]]
+local rcall = redis.call;
+
+-- Includes
+--- @include "includes/getTargetQueueList"
+
+if (rcall("EXISTS", KEYS[1]) == 1) then
+  local jobId = ARGV[1]
+  if (rcall("ZREM", KEYS[3], jobId) == 1) then
+    rcall("HDEL", KEYS[1], "finishedOn", "processedOn", ARGV[3])
+
+    local target = getTargetQueueList(KEYS[5], KEYS[4], KEYS[6])
+    rcall(ARGV[2], target, jobId)
+
+    -- Emit waiting event
+    rcall("XADD", KEYS[2], "*", "event", "waiting", "jobId", jobId, "prev", ARGV[4]);
+    return 1
+  else
+    return -3
+  end
+else
+  return -1
+end