bullmq 4.0.0-beta.1 → 4.0.0

package/dist/cjs/commands/includes/removeJobsByMaxAge.lua

@@ -0,0 +1,15 @@
+--[[
+  Functions to remove jobs by max age.
+]]
+
+-- Includes
+--- @include "removeJob"
+
+local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
+  local start = timestamp - maxAge * 1000
+  local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
+  for i, jobId in ipairs(jobIds) do
+    removeJob(jobId, false, prefix)
+  end
+  rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
+end

package/dist/cjs/commands/includes/removeJobsByMaxCount.lua

@@ -0,0 +1,15 @@
+--[[
+  Functions to remove jobs by max count.
+]]
+
+-- Includes
+--- @include "removeJob"
+
+local function removeJobsByMaxCount(maxCount, targetSet, prefix)
+  local start = maxCount
+  local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
+  for i, jobId in ipairs(jobIds) do
+    removeJob(jobId, false, prefix)
+  end
+  rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
+end

package/dist/cjs/commands/includes/removeParentDependencyKey.lua

@@ -0,0 +1,77 @@
+--[[
+  Check if this job has a parent. If so we will just remove it from
+  the parent child list, but if it is the last child we should move the parent to "wait/paused"
+  which requires code from "moveToFinished"
+]]
+
+--- @include "destructureJobKey"
+--- @include "getTargetQueueList"
+
+local function moveParentToWait(parentPrefix, parentId, emitEvent)
+  local parentTarget = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "wait", parentPrefix .. "paused")
+  rcall("RPUSH", parentTarget, parentId)
+
+  if emitEvent then
+    local parentEventStream = parentPrefix .. "events"
+    rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
+  end
+end
+
+local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey)
+  if parentKey then
+    local parentDependenciesKey = parentKey .. ":dependencies"
+    local result = rcall("SREM", parentDependenciesKey, jobKey)
+    if result > 0 then
+      local pendingDependencies = rcall("SCARD", parentDependenciesKey)
+      if pendingDependencies == 0 then
+        local parentId = getJobIdFromKey(parentKey)
+        local parentPrefix = getJobKeyPrefix(parentKey, parentId)
+
+        local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
+
+        if numRemovedElements == 1 then
+          if hard then
+            if parentPrefix == baseKey then
+              removeParentDependencyKey(parentKey, hard, nil, baseKey)
+              rcall("DEL", parentKey, parentKey .. ':logs',
+                parentKey .. ':dependencies', parentKey .. ':processed')
+            else
+              moveParentToWait(parentPrefix, parentId)
+            end
+          else
+            moveParentToWait(parentPrefix, parentId, true)
+          end
+        end
+      end
+    end
+  else
+    local missedParentKey = rcall("HGET", jobKey, "parentKey")
+    if( (type(missedParentKey) == "string") and missedParentKey ~= "" and (rcall("EXISTS", missedParentKey) == 1)) then
+      local parentDependenciesKey = missedParentKey .. ":dependencies"
+      local result = rcall("SREM", parentDependenciesKey, jobKey)
+      if result > 0 then
+        local pendingDependencies = rcall("SCARD", parentDependenciesKey)
+        if pendingDependencies == 0 then
+          local parentId = getJobIdFromKey(missedParentKey)
+          local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
+
+          local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
+
+          if numRemovedElements == 1 then
+            if hard then
+              if parentPrefix == baseKey then
+                removeParentDependencyKey(missedParentKey, hard, nil, baseKey)
+                rcall("DEL", missedParentKey, missedParentKey .. ':logs',
+                  missedParentKey .. ':dependencies', missedParentKey .. ':processed')
+              else
+                moveParentToWait(parentPrefix, parentId)
+              end
+            else
+              moveParentToWait(parentPrefix, parentId, true)
+            end
+          end
+        end
+      end
+    end
+  end
+end

package/dist/cjs/commands/includes/removeZSetJobs.lua

@@ -0,0 +1,15 @@
+-- Includes
+--- @include "batches"
+--- @include "getZSetItems"
+--- @include "removeJobs"  
+
+local function removeZSetJobs(keyName, hard, baseKey, max)
+  local jobs = getZSetItems(keyName, max)
+  local count = removeJobs(jobs, hard, baseKey, max)
+  if(#jobs > 0) then
+    for from, to in batches(#jobs, 7000) do
+      rcall("ZREM", keyName, unpack(jobs))
+    end
+  end
+  return count
+end

package/dist/cjs/commands/includes/trimEvents.lua

@@ -0,0 +1,12 @@
+--[[
+  Function to trim events, default 10000.
+]]
+
+local function trimEvents(metaKey, eventStreamKey)
+  local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
+  if maxEvents ~= false then
+    rcall("XTRIM", eventStreamKey, "MAXLEN", "~", maxEvents)
+  else
+    rcall("XTRIM", eventStreamKey, "MAXLEN", "~", 10000)
+  end
+end

package/dist/cjs/commands/includes/updateParentDepsIfNeeded.lua

@@ -0,0 +1,40 @@
+--[[
+  Validate and move or add dependencies to parent.
+]]
+
+-- Includes
+--- @include "addDelayMarkerIfNeeded"
+--- @include "addJobWithPriority"
+--- @include "getTargetQueueList"
+
+local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
+  parentId, jobIdKey, returnvalue, timestamp )
+  local processedSet = parentKey .. ":processed"
+  rcall("HSET", processedSet, jobIdKey, returnvalue)
+  local activeParent = rcall("ZSCORE", parentQueueKey .. ":waiting-children", parentId)
+  if rcall("SCARD", parentDependenciesKey) == 0 and activeParent then 
+    rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
+    local parentWaitKey = parentQueueKey .. ":wait"
+    local parentTarget, paused = getTargetQueueList(parentQueueKey .. ":meta", parentWaitKey,
+      parentQueueKey .. ":paused")
+    local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
+    local priority = tonumber(jobAttributes[1]) or 0
+    local delay = tonumber(jobAttributes[2]) or 0
+    if delay > 0 then
+      local delayedTimestamp = tonumber(timestamp) + delay 
+      local score = delayedTimestamp * 0x1000
+      local parentDelayedKey = parentQueueKey .. ":delayed" 
+      rcall("ZADD", parentDelayedKey, score, parentId)
+
+      addDelayMarkerIfNeeded(parentTarget, parentDelayedKey)
+    -- Standard or priority add
+    elseif priority == 0 then
+      rcall("RPUSH", parentTarget, parentId)
+    else
+      addJobWithPriority(parentWaitKey, parentQueueKey .. ":prioritized", priority, paused,
+        parentId, parentQueueKey .. ":pc")
+    end
+
+    rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
+  end
+end

package/dist/cjs/commands/index.js

@@ -0,0 +1,10 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.scriptLoader = exports.ScriptLoader = exports.ScriptLoaderError = void 0;
+const script_loader_1 = require("./script-loader");
+Object.defineProperty(exports, "ScriptLoader", { enumerable: true, get: function () { return script_loader_1.ScriptLoader; } });
+var script_loader_2 = require("./script-loader");
+Object.defineProperty(exports, "ScriptLoaderError", { enumerable: true, get: function () { return script_loader_2.ScriptLoaderError; } });
+const scriptLoader = new script_loader_1.ScriptLoader();
+exports.scriptLoader = scriptLoader;
+//# sourceMappingURL=index.js.map

package/dist/cjs/commands/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/commands/index.ts"],"names":[],"mappings":";;;AAAA,mDAA+C;AAKtC,6FALA,4BAAY,OAKA;AAJrB,iDAA6E;AAA3C,kHAAA,iBAAiB,OAAA;AAEnD,MAAM,YAAY,GAAG,IAAI,4BAAY,EAAE,CAAC;AAEjB,oCAAY"}

package/dist/cjs/commands/isFinished-3.lua

@@ -0,0 +1,48 @@
+--[[
+  Checks if a job is finished (.i.e. is in the completed or failed set)
+
+  Input: 
+    KEYS[1] completed key
+    KEYS[2] failed key
+    KEYS[3] job key
+
+    ARGV[1] job id
+    ARGV[2] return value?
+  Output:
+    0 - Not finished.
+    1 - Completed.
+    2 - Failed.
+   -1 - Missing job. 
+]]
+local rcall = redis.call
+if rcall("EXISTS", KEYS[3]) ~= 1 then
+  if ARGV[2] == "1" then
+
+    return {-1,"Missing key for job " .. KEYS[3] .. ". isFinished"}
+  end  
+  return -1
+end
+
+if rcall("ZSCORE", KEYS[1], ARGV[1]) ~= false then
+  if ARGV[2] == "1" then
+    local returnValue = rcall("HGET", KEYS[3], "returnvalue")
+
+    return {1,returnValue}
+  end
+  return 1
+end
+
+if rcall("ZSCORE", KEYS[2], ARGV[1]) ~= false then
+  if ARGV[2] == "1" then
+    local failedReason = rcall("HGET", KEYS[3], "failedReason")
+
+    return {2,failedReason}
+  end
+  return 2
+end
+
+if ARGV[2] == "1" then
+  return {0}
+end
+
+return 0

package/dist/cjs/commands/isJobInList-1.lua

@@ -0,0 +1,16 @@
+--[[
+  Checks if job is in a given list.
+
+  Input:
+    KEYS[1]
+    ARGV[1]
+
+  Output:
+    1 if element found in the list.
+]]
+
+-- Includes
+--- @include "includes/checkItemInList"
+
+local items = redis.call("LRANGE", KEYS[1] , 0, -1)
+return checkItemInList(items, ARGV[1])

package/dist/cjs/commands/moveJobFromActiveToWait-9.lua

@@ -0,0 +1,53 @@
+--[[
+  Function to move job from active state to wait.
+  Input:
+    KEYS[1] active key
+    KEYS[2] wait key
+    
+    KEYS[3] stalled key
+    KEYS[4] job lock key
+    KEYS[5] paused key
+    KEYS[6] meta key
+    KEYS[7] limiter key
+    KEYS[8] prioritized key
+    KEYS[9] event key
+
+    ARGV[1] job id
+    ARGV[2] lock token
+    ARGV[3] job id key
+]]
+local rcall = redis.call
+
+-- Includes
+--- @include "includes/pushBackJobWithPriority"
+--- @include "includes/getTargetQueueList"
+
+local jobId = ARGV[1]
+local token = ARGV[2]
+local lockKey = KEYS[4]
+
+local lockToken = rcall("GET", lockKey)
+local pttl = rcall("PTTL", KEYS[7])
+if lockToken == token and pttl > 0 then
+  local removed = rcall("LREM", KEYS[1], 1, jobId)
+  if (removed > 0) then
+    local target, paused = getTargetQueueList(KEYS[6], KEYS[2], KEYS[5])
+
+    rcall("SREM", KEYS[3], jobId)
+
+    local priority = tonumber(rcall("HGET", ARGV[3], "priority")) or 0
+
+    if priority > 0 then
+      pushBackJobWithPriority(KEYS[8], priority, jobId)
+    else
+      rcall("RPUSH", target, jobId)
+    end
+
+    rcall("DEL", lockKey)
+
+    -- Emit waiting event
+    rcall("XADD", KEYS[9], "*", "event", "waiting", "jobId", jobId)
+  end
+end
+
+return pttl

package/dist/cjs/commands/moveStalledJobsToWait-8.lua

@@ -0,0 +1,24 @@
+--[[
+  Move stalled jobs to wait.
+
+    Input:
+      KEYS[1] 'stalled' (SET)
+      KEYS[2] 'wait',   (LIST)
+      KEYS[3] 'active', (LIST)
+      KEYS[4] 'failed', (ZSET)
+      KEYS[5] 'stalled-check', (KEY)
+      KEYS[6] 'meta', (KEY)
+      KEYS[7] 'paused', (LIST)
+      KEYS[8] 'event stream' (STREAM)
+
+      ARGV[1]  Max stalled job count
+      ARGV[2]  queue.toKey('')
+      ARGV[3]  timestamp
+      ARGV[4]  max check time
+
+    Events:
+      'stalled' with stalled job id.
+]] -- Includes
+--- @include "includes/checkStalledJobs"
+return checkStalledJobs(KEYS[1], KEYS[2], KEYS[3], KEYS[4], KEYS[5], KEYS[6],
+                        KEYS[7], KEYS[8], ARGV[1], ARGV[2], ARGV[3], ARGV[4])

package/dist/cjs/commands/moveToActive-10.lua

@@ -0,0 +1,116 @@
+--[[
+  Move next job to be processed to active, lock it and fetch its data. The job
+  may be delayed, in that case we need to move it to the delayed set instead.
+
+  This operation guarantees that the worker owns the job during the lock
+  expiration time. The worker is responsible of keeping the lock fresh
+  so that no other worker picks this job again.
+
+  Input:
+    KEYS[1] wait key
+    KEYS[2] active key
+    KEYS[3] prioritized key
+    KEYS[4] stream events key
+    KEYS[5] stalled key
+
+    -- Rate limiting
+    KEYS[6] rate limiter key
+    KEYS[7] delayed key
+
+    -- Promote delayed jobs
+    KEYS[8] paused key
+    KEYS[9] meta key
+    KEYS[10] pc priority counter
+
+    -- Arguments
+    ARGV[1] key prefix
+    ARGV[2] timestamp
+    ARGV[3] optional job ID
+    ARGV[4] opts
+
+    opts - token - lock token
+    opts - lockDuration
+    opts - limiter
+]]
+local jobId
+local rcall = redis.call
+
+-- Includes
+--- @include "includes/getNextDelayedTimestamp"
+--- @include "includes/getRateLimitTTL"
+--- @include "includes/getTargetQueueList"
+--- @include "includes/moveJobFromPriorityToActive"
+--- @include "includes/prepareJobForProcessing"
+--- @include "includes/promoteDelayedJobs"
+
+local target, paused = getTargetQueueList(KEYS[9], KEYS[1], KEYS[8])
+
+-- Check if there are delayed jobs that we can move to wait.
+promoteDelayedJobs(KEYS[7], KEYS[1], target, KEYS[3], KEYS[4], ARGV[1], ARGV[2], paused, KEYS[10])
+
+local opts = cmsgpack.unpack(ARGV[4])
+local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
+local expireTime = getRateLimitTTL(maxJobs, KEYS[6])
+if (ARGV[3] ~= "") then
+  jobId = ARGV[3]
+  -- clean stalled key
+  rcall("SREM", KEYS[5], jobId)
+else
+  -- Check if we are rate limited first.
+  if expireTime > 0 then
+    return { 0, 0, expireTime, 0 }
+  end
+
+  -- paused queue
+  if paused then return {0, 0, 0, 0} end
+
+  -- no job ID, try non-blocking move from wait to active
+  jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
+end
+
+-- If jobId is special ID 0:delay, then there is no job to process
+if jobId then
+  if string.sub(jobId, 1, 2) == "0:" then
+    rcall("LREM", KEYS[2], 1, jobId)
+
+    if expireTime > 0 then
+      return { 0, 0, expireTime, 0 }
+    end
+
+    -- paused queue
+    if paused then return {0, 0, 0, 0} end
+
+    -- Move again since we just got the marker job.
+    jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
+
+    -- Since it is possible that between a call to BRPOPLPUSH and moveToActive
+    -- another script puts a new maker in wait, we need to check again.
+    if jobId and string.sub(jobId, 1, 2) == "0:" then
+      rcall("LREM", KEYS[2], 1, jobId)
+      jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
+    end
+  end
+
+  if jobId then
+    -- this script is not really moving, it is preparing the job for processing
+    return prepareJobForProcessing(KEYS, ARGV[1], target, jobId, ARGV[2], maxJobs, expireTime, paused, opts)
+  else
+    jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
+    if jobId then
+      return prepareJobForProcessing(KEYS, ARGV[1], target, jobId, ARGV[2], maxJobs, expireTime, paused, opts)
+    end
+  end
+else
+  jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
+  if jobId then
+    return prepareJobForProcessing(KEYS, ARGV[1], target, jobId, ARGV[2], maxJobs, expireTime, paused, opts)
+  end
+end
+
+-- Return the timestamp for the next delayed job if any.
+local nextTimestamp = getNextDelayedTimestamp(KEYS[7])
+if (nextTimestamp ~= nil) then
+  return { 0, 0, 0, nextTimestamp }
+end
+
+return { 0, 0, 0, 0}

package/dist/cjs/commands/moveToDelayed-8.lua

@@ -0,0 +1,66 @@
+--[[
+  Moves job from active to delayed set.
+
+  Input:
+    KEYS[1] wait key
+    KEYS[2] active key
+    KEYS[3] prioritized key
+    KEYS[4] delayed key
+    KEYS[5] job key
+    KEYS[6] events stream
+    KEYS[7] paused key
+    KEYS[8] meta key
+
+    ARGV[1] key prefix
+    ARGV[2] timestamp
+    ARGV[3] delayedTimestamp
+    ARGV[4] the id of the job
+    ARGV[5] queue token
+
+  Output:
+    0 - OK
+   -1 - Missing job.
+   -3 - Job not in active set.
+
+  Events:
+    - delayed key.
+]]
+local rcall = redis.call
+
+-- Includes
+--- @include "includes/addDelayMarkerIfNeeded"
+--- @include "includes/getTargetQueueList"
+--- @include "includes/promoteDelayedJobs"
+
+local jobKey = KEYS[5]
+if rcall("EXISTS", jobKey) == 1 then
+  local delayedKey = KEYS[4]
+  if ARGV[5] ~= "0" then
+    local lockKey = jobKey .. ':lock'
+    if rcall("GET", lockKey) == ARGV[5] then
+      rcall("DEL", lockKey)
+    else
+      return -2
+    end
+  end
+
+  local jobId = ARGV[4]
+  local score = tonumber(ARGV[3])
+  local delayedTimestamp = (score / 0x1000)
+
+  local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
+  if numRemovedElements < 1 then
+    return -3
+  end
+
+  rcall("ZADD", delayedKey, score, jobId)
+  rcall("XADD", KEYS[6], "*", "event", "delayed", "jobId", jobId, "delay", delayedTimestamp)
+
+  -- Check if we need to push a marker job to wake up sleeping workers.
+  local target = getTargetQueueList(KEYS[8], KEYS[1], KEYS[7])
+  addDelayMarkerIfNeeded(target, delayedKey)
+
+  return 0
+else
+  return -1
+end