bullmq 2.3.1 → 2.3.2

package/dist/esm/commands/includes/promoteDelayedJobs.lua

@@ -0,0 +1,49 @@
+--[[
+  Updates the delay set, by moving delayed jobs that should
+  be processed now to "wait".
+
+     Events:
+      'waiting'
+]]
+local rcall = redis.call
+
+-- Includes
+--- @include "addJobWithPriority"
+--- @include "getTargetQueueList"
+
+-- Try to get as much as 1000 jobs at once, and returns the nextTimestamp if
+-- there are more delayed jobs to process.
+local function promoteDelayedJobs(delayedKey, waitKey, priorityKey, pausedKey,
+                                  metaKey, eventStreamKey, prefix, timestamp)
+    local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000, "LIMIT", 0, 1000)
+
+    if (#jobs > 0) then
+        rcall("ZREM", delayedKey, unpack(jobs))
+
+        -- check if we need to use push in paused instead of waiting
+        local target = getTargetQueueList(metaKey, waitKey, pausedKey)
+
+        for _, jobId in ipairs(jobs) do
+            local priority =
+                tonumber(rcall("HGET", prefix .. jobId, "priority")) or 0
+
+            if priority == 0 then
+                -- LIFO or FIFO
+                rcall("LPUSH", target, jobId)
+            else
+                addJobWithPriority(priorityKey, priority, target, jobId)
+            end
+
+            -- Emit waiting event
+            rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
+                  jobId, "prev", "delayed")
+            rcall("HSET", prefix .. jobId, "delay", 0)
+        end
+    end
+
+    local nextTimestamp = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")[2]
+    if (nextTimestamp ~= nil) then
+        nextTimestamp = nextTimestamp / 0x1000
+    end
+    return nextTimestamp
+end

package/dist/esm/commands/includes/removeJob.lua

@@ -0,0 +1,13 @@
+--[[
+  Function to remove job.
+]]
+
+-- Includes
+--- @include "removeParentDependencyKey"
+
+local function removeJob(jobId, hard, baseKey)
+  local jobKey = baseKey .. jobId
+  removeParentDependencyKey(jobKey, hard, nil, baseKey)
+  rcall("DEL", jobKey, jobKey .. ':logs',
+    jobKey .. ':dependencies', jobKey .. ':processed')
+end

package/dist/esm/commands/includes/removeJobFromAnyState.lua

@@ -0,0 +1,26 @@
+--[[
+  Function to remove from any state.
+
+  returns:
+    prev state
+]]
+
+local function removeJobFromAnyState( prefix, jobId)
+  if rcall("LREM", prefix .. "wait", 0, jobId) == 1 then
+    return "wait"
+  elseif rcall("LREM", prefix .. "paused", 0, jobId) == 1 then
+    return "paused"
+  elseif rcall("LREM", prefix .. "active", 0, jobId) == 1 then
+    return "active"
+  elseif rcall("ZREM", prefix .. "waiting-children", jobId) == 1 then
+    return "waiting-children"
+  elseif rcall("ZREM", prefix .. "delayed", jobId) == 1 then
+    return "delayed"
+  elseif rcall("ZREM", prefix .. "completed", jobId) == 1 then
+    return "completed"
+  elseif rcall("ZREM", prefix .. "failed", jobId) == 1 then
+    return "failed"
+  end
+
+  return "unknown"
+end

package/dist/esm/commands/includes/removeJobs.lua

@@ -0,0 +1,38 @@
+--[[
+  Functions to remove jobs.
+]]
+
+-- Includes
+--- @include "batches"
+
+local function getListItems(keyName, max)
+  return rcall('LRANGE', keyName, 0, max - 1)
+end
+
+--- @include "getZSetItems"
+--- @include "removeJob"
+
+local function removeJobs(keys, hard, baseKey, max)
+  for i, key in ipairs(keys) do
+    removeJob(key, hard, baseKey)
+  end
+  return max - #keys
+end
+
+local function removeListJobs(keyName, hard, baseKey, max)
+  local jobs = getListItems(keyName, max)
+  local count = removeJobs(jobs, hard, baseKey, max)
+  rcall("LTRIM", keyName, #jobs, -1)
+  return count
+end
+
+local function removeZSetJobs(keyName, hard, baseKey, max)
+  local jobs = getZSetItems(keyName, max)
+  local count = removeJobs(jobs, hard, baseKey, max)
+  if(#jobs > 0) then
+    for from, to in batches(#jobs, 7000) do
+      rcall("ZREM", keyName, unpack(jobs))
+    end
+  end
+  return count
+end

package/dist/esm/commands/includes/removeJobsByMaxAge.lua

@@ -0,0 +1,15 @@
+--[[
+  Functions to remove jobs by max age.
+]]
+
+-- Includes
+--- @include "removeJob"
+
+local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
+  local start = timestamp - maxAge * 1000
+  local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
+  for i, jobId in ipairs(jobIds) do
+    removeJob(jobId, false, prefix)
+  end
+  rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
+end

package/dist/esm/commands/includes/removeJobsByMaxCount.lua

@@ -0,0 +1,15 @@
+--[[
+  Functions to remove jobs by max count.
+]]
+
+-- Includes
+--- @include "removeJob"
+
+local function removeJobsByMaxCount(maxCount, targetSet, prefix)
+  local start = maxCount
+  local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
+  for i, jobId in ipairs(jobIds) do
+    removeJob(jobId, false, prefix)
+  end
+  rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
+end

package/dist/esm/commands/includes/removeParentDependencyKey.lua

@@ -0,0 +1,81 @@
+--[[
+  Check if this job has a parent. If so we will just remove it from
+  the parent child list, but if it is the last child we should move the parent to "wait/paused"
+  which requires code from "moveToFinished"
+]]
+
+--- @include "destructureJobKey"
+--- @include "getTargetQueueList"
+
+local function moveParentToWait(parentPrefix, parentId, emitEvent)
+  local parentTarget = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "wait", parentPrefix .. "paused")
+  rcall("RPUSH", parentTarget, parentId)
+
+  if emitEvent then
+    local parentEventStream = parentPrefix .. "events"
+    rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
+  end
+end
+
+local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
+  if parentKey then
+    local parentProcessedKey = parentKey .. ":processed"
+    rcall("HDEL", parentProcessedKey, jobKey)
+    local parentDependenciesKey = parentKey .. ":dependencies"
+    local result = rcall("SREM", parentDependenciesKey, jobKey)
+    if result > 0 then
+      local pendingDependencies = rcall("SCARD", parentDependenciesKey)
+      if pendingDependencies == 0 then
+        local parentId = getJobIdFromKey(parentKey)
+        local parentPrefix = getJobKeyPrefix(parentKey, parentId)
+
+        local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
+
+        if numRemovedElements == 1 then
+          if hard then
+            if parentPrefix == baseKey then
+              removeParentDependencyKey(parentKey, hard, nil, baseKey)
+              rcall("DEL", parentKey, parentKey .. ':logs',
+                parentKey .. ':dependencies', parentKey .. ':processed')
+            else
+              moveParentToWait(parentPrefix, parentId)
+            end
+          else
+            moveParentToWait(parentPrefix, parentId, true)
+          end
+        end
+      end
+    end
+  else
+    local missedParentKey = rcall("HGET", jobKey, "parentKey")
+    if( (type(missedParentKey) == "string") and missedParentKey ~= "" and (rcall("EXISTS", missedParentKey) == 1)) then
+      local parentProcessedKey = missedParentKey .. ":processed"
+      rcall("HDEL", parentProcessedKey, jobKey)
+      local parentDependenciesKey = missedParentKey .. ":dependencies"
+      local result = rcall("SREM", parentDependenciesKey, jobKey)
+      if result > 0 then
+        local pendingDependencies = rcall("SCARD", parentDependenciesKey)
+        if pendingDependencies == 0 then
+          local parentId = getJobIdFromKey(missedParentKey)
+          local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
+
+          local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
+
+          if numRemovedElements == 1 then
+            if hard then
+              if parentPrefix == baseKey then
+                removeParentDependencyKey(missedParentKey, hard, nil, baseKey)
+                rcall("DEL", missedParentKey, missedParentKey .. ':logs',
+                  missedParentKey .. ':dependencies', missedParentKey .. ':processed')
+              else
+                moveParentToWait(parentPrefix, parentId)
+              end
+            else
+              moveParentToWait(parentPrefix, parentId, true)
+            end
+          end
+        end
+      end
+    end
+  end
+end

package/dist/esm/commands/includes/trimEvents.lua

@@ -0,0 +1,12 @@
+--[[
+  Function to trim events, default 10000.
+]]
+
+local function trimEvents(metaKey, eventStreamKey)
+  local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
+  if maxEvents ~= false then
+    rcall("XTRIM", eventStreamKey, "MAXLEN", "~", maxEvents)
+  else
+    rcall("XTRIM", eventStreamKey, "MAXLEN", "~", 10000)
+  end
+end

package/dist/esm/commands/includes/updateParentDepsIfNeeded.lua

@@ -0,0 +1,28 @@
+--[[
+  Validate and move or add dependencies to parent.
+]]
+
+-- Includes
+--- @include "addJobWithPriority"
+--- @include "getTargetQueueList"
+
+local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
+  parentId, jobIdKey, returnvalue )
+  local processedSet = parentKey .. ":processed"
+  rcall("HSET", processedSet, jobIdKey, returnvalue)
+  local activeParent = rcall("ZSCORE", parentQueueKey .. ":waiting-children", parentId)
+  if rcall("SCARD", parentDependenciesKey) == 0 and activeParent then 
+    rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
+    local parentTarget = getTargetQueueList(parentQueueKey .. ":meta", parentQueueKey .. ":wait",
+      parentQueueKey .. ":paused")
+    local priority = tonumber(rcall("HGET", parentKey, "priority"))
+    -- Standard or priority add
+    if priority == 0 then
+      rcall("RPUSH", parentTarget, parentId)
+    else
+      addJobWithPriority(parentQueueKey .. ":priority", priority, parentTarget, parentId)
+    end
+
+    rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
+  end
+end

package/dist/esm/commands/isFinished-3.lua

@@ -0,0 +1,48 @@
+--[[
+  Checks if a job is finished (.i.e. is in the completed or failed set)
+
+  Input: 
+    KEYS[1] completed key
+    KEYS[2] failed key
+    KEYS[3] job key
+
+    ARGV[1] job id
+    ARGV[2] return value?
+  Output:
+    0 - Not finished.
+    1 - Completed.
+    2 - Failed.
+   -5 - Missing job. 
+]]
+local rcall = redis.call
+if rcall("EXISTS", KEYS[3]) ~= 1 then
+  if ARGV[2] == "1" then
+
+    return {-5,"Missing key for job " .. KEYS[3] .. ". isFinished"}
+  end  
+  return -5
+end
+
+if rcall("ZSCORE", KEYS[1], ARGV[1]) ~= false then
+  if ARGV[2] == "1" then
+    local returnValue = rcall("HGET", KEYS[3], "returnvalue")
+
+    return {1,returnValue}
+  end
+  return 1
+end
+
+if rcall("ZSCORE", KEYS[2], ARGV[1]) ~= false then
+  if ARGV[2] == "1" then
+    local failedReason = rcall("HGET", KEYS[3], "failedReason")
+
+    return {2,failedReason}
+  end
+  return 2
+end
+
+if ARGV[2] == "1" then
+  return {0}
+end
+
+return 0

package/dist/esm/commands/isJobInList-1.lua

@@ -0,0 +1,16 @@
+--[[
+  Checks if job is in a given list.
+
+  Input:
+    KEYS[1]
+    ARGV[1]
+
+  Output:
+    1 if element found in the list.
+]]
+
+-- Includes
+--- @include "includes/checkItemInList"
+
+local items = redis.call("LRANGE", KEYS[1] , 0, -1)
+return checkItemInList(items, ARGV[1])

package/dist/esm/commands/moveStalledJobsToWait-8.lua

@@ -0,0 +1,24 @@
+--[[
+  Move stalled jobs to wait.
+
+    Input:
+      KEYS[1] 'stalled' (SET)
+      KEYS[2] 'wait',   (LIST)
+      KEYS[3] 'active', (LIST)
+      KEYS[4] 'failed', (ZSET)
+      KEYS[5] 'stalled-check', (KEY)
+      KEYS[6] 'meta', (KEY)
+      KEYS[7] 'paused', (LIST)
+      KEYS[8] 'event stream' (STREAM)
+
+      ARGV[1]  Max stalled job count
+      ARGV[2]  queue.toKey('')
+      ARGV[3]  timestamp
+      ARGV[4]  max check time
+
+    Events:
+      'stalled' with stalled job id.
+]] -- Includes
+--- @include "includes/checkStalledJobs"
+return checkStalledJobs(KEYS[1], KEYS[2], KEYS[3], KEYS[4], KEYS[5], KEYS[6],
+                        KEYS[7], KEYS[8], ARGV[1], ARGV[2], ARGV[3], ARGV[4])

package/dist/esm/commands/moveToActive-9.lua

@@ -0,0 +1,67 @@
+--[[
+  Move next job to be processed to active, lock it and fetch its data. The job
+  may be delayed, in that case we need to move it to the delayed set instead.
+
+  This operation guarantees that the worker owns the job during the lock
+  expiration time. The worker is responsible of keeping the lock fresh
+  so that no other worker picks this job again.
+
+  Input:
+      KEYS[1] wait key
+      KEYS[2] active key
+      KEYS[3] priority key
+      KEYS[4] stream events key
+      KEYS[5] stalled key
+
+      -- Rate limiting
+      KEYS[6] rate limiter key
+      KEYS[7] delayed key
+
+      -- Promote delayed jobs
+      KEYS[8] paused key
+      KEYS[9] meta key
+
+      -- Arguments
+      ARGV[1] key prefix
+      ARGV[2] timestamp
+      ARGV[3] optional job ID
+      ARGV[4] opts
+
+      opts - token - lock token
+      opts - lockDuration
+      opts - limiter
+]]
+local jobId
+local rcall = redis.call
+
+-- Includes
+--- @include "includes/moveJobFromWaitToActive"
+--- @include "includes/getNextDelayedTimestamp"
+--- @include "includes/promoteDelayedJobs"
+
+-- Check if there are delayed jobs that we can move to wait.
+promoteDelayedJobs(KEYS[7], KEYS[1], KEYS[3], KEYS[8], KEYS[9], KEYS[4], ARGV[1], ARGV[2])
+
+if (ARGV[3] ~= "") then
+    jobId = ARGV[3]
+    -- clean stalled key
+    rcall("SREM", KEYS[5], jobId)
+else
+    -- no job ID, try non-blocking move from wait to active
+    jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
+end
+
+-- If jobId is special ID 0, then there is no job to process
+if jobId == "0" then
+    rcall("LREM", KEYS[2], 1, 0)
+elseif jobId then
+    local opts = cmsgpack.unpack(ARGV[4])
+    -- this script is not really moving, it is preparing the job for processing
+    return moveJobFromWaitToActive(KEYS, ARGV[1], jobId, ARGV[2], opts)
+end
+
+-- Return the timestamp for the next delayed job if any.
+local nextTimestamp = getNextDelayedTimestamp(KEYS[7])
+if (nextTimestamp ~= nil) then
+    return nextTimestamp - tonumber(ARGV[2])
+end

package/dist/esm/commands/moveToDelayed-5.lua

@@ -0,0 +1,54 @@
+--[[
+  Moves job from active to delayed set.
+
+  Input: 
+    KEYS[1] active key
+    KEYS[2] delayed key
+    KEYS[3] job key
+    KEYS[4] events stream
+    KEYS[5] delayed stream
+
+    ARGV[1] delayedTimestamp
+    ARGV[2] the id of the job
+    ARGV[3] queue token
+
+  Output:
+    0 - OK
+   -1 - Missing job.
+   -3 - Job not in active set.
+
+  Events:
+    - delayed key.
+]]
+local rcall = redis.call
+
+if rcall("EXISTS", KEYS[3]) == 1 then
+
+  if ARGV[3] ~= "0" then
+    local lockKey = KEYS[3] .. ':lock'
+    if rcall("GET", lockKey) == ARGV[3] then
+      rcall("DEL", lockKey)
+    else
+      return -2
+    end
+  end
+
+  local jobId = ARGV[2]
+  local score = tonumber(ARGV[1])
+  local delayedTimestamp = (score / 0x1000)
+  
+  local numRemovedElements = rcall("LREM", KEYS[1], -1, jobId)
+
+  if(numRemovedElements < 1) then
+    return -3
+  end
+
+  rcall("ZADD", KEYS[2], score, jobId)
+
+  rcall("XADD", KEYS[4], "*", "event", "delayed", "jobId", jobId, "delay", delayedTimestamp);
+  rcall("XADD", KEYS[5], "*", "nextTimestamp", delayedTimestamp);
+
+  return 0
+else
+  return -1
+end