bullmq 2.3.1 → 2.4.0

package/dist/esm/commands/moveStalledJobsToWait-8.lua

@@ -0,0 +1,24 @@
+--[[
+  Move stalled jobs to wait.
+
+    Input:
+      KEYS[1] 'stalled' (SET)
+      KEYS[2] 'wait',   (LIST)
+      KEYS[3] 'active', (LIST)
+      KEYS[4] 'failed', (ZSET)
+      KEYS[5] 'stalled-check', (KEY)
+      KEYS[6] 'meta', (KEY)
+      KEYS[7] 'paused', (LIST)
+      KEYS[8] 'event stream' (STREAM)
+
+      ARGV[1]  Max stalled job count
+      ARGV[2]  queue.toKey('')
+      ARGV[3]  timestamp
+      ARGV[4]  max check time
+
+    Events:
+      'stalled' with stalled job id.
+]] -- Includes
+--- @include "includes/checkStalledJobs"
+return checkStalledJobs(KEYS[1], KEYS[2], KEYS[3], KEYS[4], KEYS[5], KEYS[6],
+                        KEYS[7], KEYS[8], ARGV[1], ARGV[2], ARGV[3], ARGV[4])

package/dist/esm/commands/moveToActive-9.lua

@@ -0,0 +1,67 @@
+--[[
+  Move next job to be processed to active, lock it and fetch its data. The job
+  may be delayed, in that case we need to move it to the delayed set instead.
+
+  This operation guarantees that the worker owns the job during the lock
+  expiration time. The worker is responsible of keeping the lock fresh
+  so that no other worker picks this job again.
+
+  Input:
+      KEYS[1] wait key
+      KEYS[2] active key
+      KEYS[3] priority key
+      KEYS[4] stream events key
+      KEYS[5] stalled key
+
+      -- Rate limiting
+      KEYS[6] rate limiter key
+      KEYS[7] delayed key
+
+      -- Promote delayed jobs
+      KEYS[8] paused key
+      KEYS[9] meta key
+
+      -- Arguments
+      ARGV[1] key prefix
+      ARGV[2] timestamp
+      ARGV[3] optional job ID
+      ARGV[4] opts
+
+      opts - token - lock token
+      opts - lockDuration
+      opts - limiter
+]]
+local jobId
+local rcall = redis.call
+
+-- Includes
+--- @include "includes/moveJobFromWaitToActive"
+--- @include "includes/getNextDelayedTimestamp"
+--- @include "includes/promoteDelayedJobs"
+
+-- Check if there are delayed jobs that we can move to wait.
+promoteDelayedJobs(KEYS[7], KEYS[1], KEYS[3], KEYS[8], KEYS[9], KEYS[4], ARGV[1], ARGV[2])
+
+if (ARGV[3] ~= "") then
+    jobId = ARGV[3]
+    -- clean stalled key
+    rcall("SREM", KEYS[5], jobId)
+else
+    -- no job ID, try non-blocking move from wait to active
+    jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
+end
+
+-- If jobId is special ID 0, then there is no job to process
+if jobId == "0" then
+    rcall("LREM", KEYS[2], 1, 0)
+elseif jobId then
+    local opts = cmsgpack.unpack(ARGV[4])
+    -- this script is not really moving, it is preparing the job for processing
+    return moveJobFromWaitToActive(KEYS, ARGV[1], jobId, ARGV[2], opts)
+end
+
+-- Return the timestamp for the next delayed job if any.
+local nextTimestamp = getNextDelayedTimestamp(KEYS[7])
+if (nextTimestamp ~= nil) then
+    return nextTimestamp - tonumber(ARGV[2])
+end

package/dist/esm/commands/moveToDelayed-5.lua

@@ -0,0 +1,54 @@
+--[[
+  Moves job from active to delayed set.
+
+  Input: 
+    KEYS[1] active key
+    KEYS[2] delayed key
+    KEYS[3] job key
+    KEYS[4] events stream
+    KEYS[5] delayed stream
+
+    ARGV[1] delayedTimestamp
+    ARGV[2] the id of the job
+    ARGV[3] queue token
+
+  Output:
+    0 - OK
+   -1 - Missing job.
+   -3 - Job not in active set.
+
+  Events:
+    - delayed key.
+]]
+local rcall = redis.call
+
+if rcall("EXISTS", KEYS[3]) == 1 then
+
+  if ARGV[3] ~= "0" then
+    local lockKey = KEYS[3] .. ':lock'
+    if rcall("GET", lockKey) == ARGV[3] then
+      rcall("DEL", lockKey)
+    else
+      return -2
+    end
+  end
+
+  local jobId = ARGV[2]
+  local score = tonumber(ARGV[1])
+  local delayedTimestamp = (score / 0x1000)
+  
+  local numRemovedElements = rcall("LREM", KEYS[1], -1, jobId)
+
+  if(numRemovedElements < 1) then
+    return -3
+  end
+
+  rcall("ZADD", KEYS[2], score, jobId)
+
+  rcall("XADD", KEYS[4], "*", "event", "delayed", "jobId", jobId, "delay", delayedTimestamp);
+  rcall("XADD", KEYS[5], "*", "nextTimestamp", delayedTimestamp);
+
+  return 0
+else
+  return -1
+end

package/dist/esm/commands/moveToFinished-12.lua

@@ -0,0 +1,201 @@
+--[[
+  Move job from active to a finished status (completed o failed)
+  A job can only be moved to completed if it was active.
+  The job must be locked before it can be moved to a finished status,
+  and the lock must be released in this script.
+
+    Input:
+      KEYS[1] wait key
+      KEYS[2] active key
+      KEYS[3] priority key
+      KEYS[4] event stream key
+      KEYS[5] stalled key
+
+      -- Rate limiting
+      KEYS[6] rate limiter key
+      KEYS[7] delayed key
+
+      KEYS[8] paused key
+
+      KEYS[9] completed/failed key
+      KEYS[10] jobId key
+      KEYS[11] meta key
+      KEYS[12] metrics key
+
+      ARGV[1]  jobId
+      ARGV[2]  timestamp
+      ARGV[3]  msg property returnvalue / failedReason
+      ARGV[4]  return value / failed reason
+      ARGV[5]  target (completed/failed)
+      ARGV[6]  event data (? maybe just send jobid).
+      ARGV[7]  fetch next?
+      ARGV[8]  keys prefix
+      ARGV[9]  opts
+
+      opts - token - lock token
+      opts - keepJobs
+      opts - lockDuration - lock duration in milliseconds
+      opts - parent - parent data
+      opts - parentKey
+      opts - attempts max attempts
+      opts - attemptsMade
+      opts - maxMetricsSize
+      opts - fpof - fail parent on fail
+
+    Output:
+      0 OK
+      -1 Missing key.
+      -2 Missing lock.
+      -3 Job not in active set
+      -4 Job has pending dependencies
+
+    Events:
+      'completed/failed'
+]]
+local rcall = redis.call
+
+--- Includes
+--- @include "includes/collectMetrics"
+--- @include "includes/getNextDelayedTimestamp"
+--- @include "includes/moveJobFromWaitToActive"
+--- @include "includes/moveParentFromWaitingChildrenToFailed"
+--- @include "includes/promoteDelayedJobs"
+--- @include "includes/removeJobsByMaxAge"
+--- @include "includes/removeJobsByMaxCount"
+--- @include "includes/trimEvents"
+--- @include "includes/updateParentDepsIfNeeded"
+
+local jobIdKey = KEYS[10]
+if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
+    local opts = cmsgpack.unpack(ARGV[9])
+
+    local token = opts['token']
+    local parentId = opts['parent'] and opts['parent']['id'] or ""
+    local parentQueueKey = opts['parent'] and opts['parent']['queue'] or ""
+    local parentKey = opts['parentKey'] or ""
+    local attempts = opts['attempts']
+    local attemptsMade = opts['attemptsMade']
+    local maxMetricsSize = opts['maxMetricsSize']
+    local maxCount = opts['keepJobs']['count']
+    local maxAge = opts['keepJobs']['age']
+
+    if token ~= "0" then
+        local lockKey = jobIdKey .. ':lock'
+        if rcall("GET", lockKey) == token then
+            rcall("DEL", lockKey)
+            rcall("SREM", KEYS[5], ARGV[1])
+        else
+            return -2
+        end
+    end
+
+    if rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then -- // Make sure it does not have pending dependencies
+        return -4
+    end
+
+    local jobId = ARGV[1]
+    local timestamp = ARGV[2]
+
+    -- Remove from active list (if not active we shall return error)
+    local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
+
+    if (numRemovedElements < 1) then return -3 end
+
+    -- Trim events before emiting them to avoid trimming events emitted in this script
+    trimEvents(KEYS[11], KEYS[4])
+
+    -- If job has a parent we need to
+    -- 1) remove this job id from parents dependencies
+    -- 2) move the job Id to parent "processed" set
+    -- 3) push the results into parent "results" list
+    -- 4) if parent's dependencies is empty, then move parent to "wait/paused". Note it may be a different queue!.
+    if parentId == "" and parentKey ~= "" then
+        parentId = getJobIdFromKey(parentKey)
+        parentQueueKey = getJobKeyPrefix(parentKey, ":" .. parentId)
+    end
+    if parentId ~= "" then
+        if ARGV[5] == "completed" then
+            local dependenciesSet = parentKey .. ":dependencies"
+            if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
+                updateParentDepsIfNeeded(parentKey, parentQueueKey, dependenciesSet,
+                                        parentId, jobIdKey, ARGV[4])
+            end
+        elseif opts['fpof'] then
+            moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
+        end
+    end
+
+    -- Remove job?
+    if maxCount ~= 0 then
+        local targetSet = KEYS[9]
+        -- Add to complete/failed set
+        rcall("ZADD", targetSet, timestamp, jobId)
+        rcall("HMSET", jobIdKey, ARGV[3], ARGV[4], "finishedOn", timestamp)
+        -- "returnvalue" / "failedReason" and "finishedOn"
+
+        -- Remove old jobs?
+        local prefix = ARGV[8]
+
+        if maxAge ~= nil then
+            removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
+        end
+
+        if maxCount ~= nil and maxCount > 0 then
+            removeJobsByMaxCount(maxCount, targetSet, prefix)
+        end
+    else
+        rcall("DEL", jobIdKey, jobIdKey .. ':logs', jobIdKey .. ':processed')
+    end
+
+    rcall("XADD", KEYS[4], "*", "event", ARGV[5], "jobId", jobId, ARGV[3],
+          ARGV[4])
+
+    if ARGV[5] == "failed" then
+        if tonumber(attemptsMade) >= tonumber(attempts) then
+            rcall("XADD", KEYS[4], "*", "event", "retries-exhausted", "jobId",
+                  jobId, "attemptsMade", attemptsMade)
+        end
+    end
+
+    -- Collect metrics
+    if maxMetricsSize ~= "" then
+        collectMetrics(KEYS[12], KEYS[12]..':data', maxMetricsSize, timestamp)
+    end
+
+    -- Try to get next job to avoid an extra roundtrip if the queue is not closing,
+    -- and not rate limited.
+    if (ARGV[7] == "1") then
+
+        -- Check if there are delayed jobs that can be promoted
+        promoteDelayedJobs(KEYS[7], KEYS[1], KEYS[3], KEYS[8], KEYS[11], KEYS[4], ARGV[8], timestamp)
+
+        jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
+
+        if jobId == "0" then
+            rcall("LREM", KEYS[2], 1, 0)
+        elseif jobId then
+            return moveJobFromWaitToActive(KEYS, ARGV[8], jobId, timestamp, opts)
+        end
+
+        -- Return the timestamp for the next delayed job if any.
+        local nextTimestamp = getNextDelayedTimestamp(KEYS[7])
+        if (nextTimestamp ~= nil) then
+            -- The result is guaranteed to be positive, since the
+            -- ZRANGEBYSCORE command would have return a job otherwise.
+            return nextTimestamp - timestamp
+        end
+    end
+
+    local waitLen = rcall("LLEN", KEYS[1])
+    if waitLen == 0 then
+        local activeLen = rcall("LLEN", KEYS[2])
+
+        if activeLen == 0 then
+            rcall("XADD", KEYS[4], "*", "event", "drained")
+        end
+    end  
+
+    return 0
+else
+    return -1
+end

package/dist/esm/commands/moveToWaitingChildren-4.lua

@@ -0,0 +1,62 @@
+--[[
+  Moves job from active to waiting children set.
+
+  Input:
+    KEYS[1] lock key
+    KEYS[2] active key
+    KEYS[3] waitChildrenKey key
+    KEYS[4] job key
+
+    ARGV[1] token
+    ARGV[2] child key
+    ARGV[3] timestamp
+    ARGV[4] the id of the job
+    
+  Output:
+    0 - OK
+    1 - There are not pending dependencies.
+   -1 - Missing job.
+   -2 - Missing lock
+   -3 - Job not in active set
+]]
+local rcall = redis.call
+
+local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId, timestamp, lockKey, token)
+  if token ~= "0" then
+    if rcall("GET", lockKey) == token then
+      rcall("DEL", lockKey)
+    else
+      return -2
+    end
+  end
+
+  local score = tonumber(timestamp)
+
+  local numRemovedElements = rcall("LREM", activeKey, -1, jobId)
+
+  if(numRemovedElements < 1) then
+    return -3
+  end
+
+  rcall("ZADD", waitingChildrenKey, score, jobId)
+
+  return 0
+end
+
+if rcall("EXISTS", KEYS[4]) == 1 then
+  if ARGV[2] ~= "" then
+    if rcall("SISMEMBER", KEYS[4] .. ":dependencies", ARGV[2]) ~= 0 then
+      return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], ARGV[1])
+    end
+
+    return 1
+  else
+    if rcall("SCARD", KEYS[4] .. ":dependencies") ~= 0 then 
+      return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3], KEYS[1], ARGV[1])
+    end
+
+    return 1
+  end
+end
+
+return -1

package/dist/esm/commands/obliterate-2.lua

@@ -0,0 +1,94 @@
+--[[
+  Completely obliterates a queue and all of its contents
+  
+  Input:
+    KEYS[1] meta
+    KEYS[2] base
+
+    ARGV[1] count
+    ARGV[2] force
+]]
+
+-- This command completely destroys a queue including all of its jobs, current or past 
+-- leaving no trace of its existence. Since this script needs to iterate to find all the job
+-- keys, consider that this call may be slow for very large queues.
+
+-- The queue needs to be "paused" or it will return an error
+-- If the queue has currently active jobs then the script by default will return error,
+-- however this behaviour can be overrided using the 'force' option.
+local maxCount = tonumber(ARGV[1])
+local baseKey = KEYS[2]
+
+local rcall = redis.call
+
+--- @include "includes/removeJobs"
+
+local function removeLockKeys(keys)
+  for i, key in ipairs(keys) do
+    rcall("DEL", baseKey .. key .. ':lock')
+  end
+end
+
+-- 1) Check if paused, if not return with error.
+if rcall("HEXISTS", KEYS[1], "paused") ~= 1 then
+  return -1 -- Error, NotPaused
+end
+
+-- 2) Check if there are active jobs, if there are and not "force" return error.
+local activeKey = baseKey .. 'active'
+local activeJobs = getListItems(activeKey, maxCount)
+if (#activeJobs > 0) then
+  if(ARGV[2] == "") then 
+    return -2 -- Error, ExistActiveJobs
+  end
+end
+
+removeLockKeys(activeJobs)
+maxCount = removeJobs(activeJobs, true, baseKey, maxCount)
+rcall("LTRIM", activeKey, #activeJobs, -1)
+if(maxCount <= 0) then
+  return 1
+end
+
+local delayedKey = baseKey .. 'delayed'
+maxCount = removeZSetJobs(delayedKey, true, baseKey, maxCount)
+if(maxCount <= 0) then
+  return 1
+end
+
+local completedKey = baseKey .. 'completed'
+maxCount = removeZSetJobs(completedKey, true, baseKey, maxCount)
+if(maxCount <= 0) then
+  return 1
+end
+
+local waitKey = baseKey .. 'paused'
+maxCount = removeListJobs(waitKey, true, baseKey, maxCount)
+if(maxCount <= 0) then
+  return 1
+end
+
+local failedKey = baseKey .. 'failed'
+maxCount = removeZSetJobs(failedKey, true, baseKey, maxCount)
+if(maxCount <= 0) then
+  return 1
+end
+
+if(maxCount > 0) then
+  rcall("DEL", 
+    baseKey .. 'priority',
+    baseKey .. 'events',
+    baseKey .. 'delay', 
+    baseKey .. 'stalled-check',
+    baseKey .. 'stalled',
+    baseKey .. 'id',
+    baseKey .. 'meta',
+    baseKey .. 'repeat',
+    baseKey .. 'metrics:completed',
+    baseKey .. 'metrics:completed:data',
+    baseKey .. 'metrics:failed',
+    baseKey .. 'metrics:failed:data')
+  return 0
+else
+  return 1
+end

package/dist/esm/commands/pause-4.lua

@@ -0,0 +1,27 @@
+--[[
+  Pauses or resumes a queue globably.
+
+   Input:
+      KEYS[1] 'wait' or 'paused''
+      KEYS[2] 'paused' or 'wait'
+      KEYS[3] 'meta'
+      KEYS[4] events stream key
+
+      ARGV[1] 'paused' or 'resumed'
+
+    Event:
+      publish paused or resumed event.
+]]
+local rcall = redis.call
+
+if rcall("EXISTS", KEYS[1]) == 1 then
+  rcall("RENAME", KEYS[1], KEYS[2])
+end
+
+if ARGV[1] == "paused" then
+  rcall("HSET", KEYS[3], "paused", 1)
+else
+  rcall("HDEL", KEYS[3], "paused")
+end
+
+rcall("XADD", KEYS[4], "*", "event", ARGV[1]);

package/dist/esm/commands/promote-6.lua

@@ -0,0 +1,51 @@
+--[[
+  Promotes a job that is currently "delayed" to the "waiting" state
+
+     Input:
+      KEYS[1] 'delayed'
+      KEYS[2] 'wait'
+      KEYS[3] 'paused'
+      KEYS[4] 'meta'
+      KEYS[5] 'priority'
+      KEYS[6] 'event stream'
+
+      ARGV[1]  queue.toKey('')
+      ARGV[2]  jobId
+
+     Events:
+      'waiting'
+]]
+local rcall = redis.call;
+local jobId = ARGV[2]
+
+-- Includes
+--- @include "includes/addJobWithPriority"
+--- @include "includes/getTargetQueueList"
+
+if rcall("ZREM", KEYS[1], jobId) == 1 then
+  local priority = tonumber(rcall("HGET", ARGV[1] .. jobId, "priority")) or 0
+  local target = getTargetQueueList(KEYS[4], KEYS[2], KEYS[3])
+
+  -- Remove delayed "marker" from the wait list if there is any.
+  -- Since we are adding a job we do not need the marker anymore.
+  if rcall("LINDEX", target, 0) == "0" then
+    rcall("LPOP", target)
+  end
+
+  if priority == 0 then
+    -- LIFO or FIFO
+    rcall("LPUSH", target, jobId)
+  else
+    -- Priority add
+    addJobWithPriority(KEYS[5], priority, target, jobId)
+  end
+
+  -- Emit waiting event (wait..ing@token)
+  rcall("XADD", KEYS[6], "*", "event", "waiting", "jobId", jobId, "prev", "delayed");
+
+  rcall("HSET", ARGV[1] .. jobId, "delay", 0)
+
+  return 0
+else
+  return -3
+end

package/dist/esm/commands/releaseLock-1.lua

@@ -0,0 +1,19 @@
+--[[
+  Release lock
+
+    Input:
+      KEYS[1] 'lock',
+    
+      ARGV[1]  token
+      ARGV[2]  lock duration in milliseconds
+      
+    Output:
+      "OK" if lock extented succesfully.
+]]
+local rcall = redis.call
+
+if rcall("GET", KEYS[1]) == ARGV[1] then
+  return rcall("DEL", KEYS[1])
+else
+  return 0
+end

package/dist/esm/commands/removeJob-1.lua

@@ -0,0 +1,72 @@
+--[[
+    Remove a job from all the queues it may be in as well as all its data.
+    In order to be able to remove a job, it cannot be active.
+
+    Input:
+      KEYS[1] queue prefix
+      ARGV[1] jobId
+
+    Events:
+      'removed'
+]]
+
+local rcall = redis.call
+
+-- Includes
+--- @include "includes/destructureJobKey"
+--- @include "includes/isLocked"
+--- @include "includes/removeJobFromAnyState"
+--- @include "includes/removeParentDependencyKey"
+
+local function removeJob( prefix, jobId, parentKey)
+    local jobKey = prefix .. jobId;
+
+    removeParentDependencyKey(jobKey, false, parentKey)
+
+    -- Check if this job has children
+    -- If so, we are going to try to remove the children recursively in deep first way because
+    -- if some job is locked we must exit with and error.
+    --local countProcessed = rcall("HLEN", jobKey .. ":processed")
+    local processed = rcall("HGETALL", jobKey .. ":processed")
+
+    if (#processed > 0) then
+        for i = 1, #processed, 2 do
+            local childJobId = getJobIdFromKey(processed[i])
+            local childJobPrefix = getJobKeyPrefix(processed[i], childJobId)
+            removeJob( childJobPrefix, childJobId, jobKey )
+        end
+    end
+
+    local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
+    if (#dependencies > 0) then
+        for i, childJobKey in ipairs(dependencies) do
+            -- We need to get the jobId for this job.
+            local childJobId = getJobIdFromKey(childJobKey)
+            local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
+            removeJob( childJobPrefix, childJobId, jobKey )
+        end
+    end
+
+    local prev = removeJobFromAnyState(prefix, jobId)
+
+    rcall("ZREM", prefix .. "priority", jobId)
+    rcall("DEL", jobKey, jobKey .. ":logs", jobKey .. ":dependencies", jobKey .. ":processed")
+
+    -- -- delete keys related to rate limiter
+        -- local limiterIndexTable = KEYS[10] .. ":index"
+        -- local limitedSetKey = rcall("HGET", limiterIndexTable, jobId)
+        -- if limitedSetKey then
+        --     rcall("SREM", limitedSetKey, jobId)
+        --     rcall("HDEL", limiterIndexTable, jobId)
+    -- end
+
+    rcall("XADD", prefix .. "events", "*", "event", "removed", "jobId", jobId, "prev", prev);
+end
+
+local prefix = KEYS[1]
+
+if not isLocked(prefix, ARGV[1]) then
+    removeJob(prefix, ARGV[1])
+    return 1
+end
+return 0