bullmq 2.2.0 → 2.3.0

package/dist/esm/commands/moveToFinished-12.lua

@@ -1,201 +0,0 @@
---[[
-  Move job from active to a finished status (completed o failed)
-  A job can only be moved to completed if it was active.
-  The job must be locked before it can be moved to a finished status,
-  and the lock must be released in this script.
-
-    Input:
-      KEYS[1] wait key
-      KEYS[2] active key
-      KEYS[3] priority key
-      KEYS[4] event stream key
-      KEYS[5] stalled key
-
-      -- Rate limiting
-      KEYS[6] rate limiter key
-      KEYS[7] delayed key
-
-      KEYS[8] paused key
-
-      KEYS[9] completed/failed key
-      KEYS[10] jobId key
-      KEYS[11] meta key
-      KEYS[12] metrics key
-
-      ARGV[1]  jobId
-      ARGV[2]  timestamp
-      ARGV[3]  msg property returnvalue / failedReason
-      ARGV[4]  return value / failed reason
-      ARGV[5]  target (completed/failed)
-      ARGV[6]  event data (? maybe just send jobid).
-      ARGV[7]  fetch next?
-      ARGV[8]  keys prefix
-      ARGV[9]  opts
-
-      opts - token - lock token
-      opts - keepJobs
-      opts - lockDuration - lock duration in milliseconds
-      opts - parent - parent data
-      opts - parentKey
-      opts - attempts max attempts
-      opts - attemptsMade
-      opts - maxMetricsSize
-      opts - fpof - fail parent on fail
-
-    Output:
-      0 OK
-      -1 Missing key.
-      -2 Missing lock.
-      -3 Job not in active set
-      -4 Job has pending dependencies
-
-    Events:
-      'completed/failed'
-]]
-local rcall = redis.call
-
---- Includes
---- @include "includes/collectMetrics"
---- @include "includes/getNextDelayedTimestamp"
---- @include "includes/moveJobFromWaitToActive"
---- @include "includes/moveParentFromWaitingChildrenToFailed"
---- @include "includes/promoteDelayedJobs"
---- @include "includes/removeJobsByMaxAge"
---- @include "includes/removeJobsByMaxCount"
---- @include "includes/trimEvents"
---- @include "includes/updateParentDepsIfNeeded"
-
-local jobIdKey = KEYS[10]
-if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
-    local opts = cmsgpack.unpack(ARGV[9])
-
-    local token = opts['token']
-    local parentId = opts['parent'] and opts['parent']['id'] or ""
-    local parentQueueKey = opts['parent'] and opts['parent']['queue'] or ""
-    local parentKey = opts['parentKey'] or ""
-    local attempts = opts['attempts']
-    local attemptsMade = opts['attemptsMade']
-    local maxMetricsSize = opts['maxMetricsSize']
-    local maxCount = opts['keepJobs']['count']
-    local maxAge = opts['keepJobs']['age']
-
-    if token ~= "0" then
-        local lockKey = jobIdKey .. ':lock'
-        if rcall("GET", lockKey) == token then
-            rcall("DEL", lockKey)
-            rcall("SREM", KEYS[5], ARGV[1])
-        else
-            return -2
-        end
-    end
-
-    if rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then -- // Make sure it does not have pending dependencies
-        return -4
-    end
-
-    local jobId = ARGV[1]
-    local timestamp = ARGV[2]
-
-    -- Remove from active list (if not active we shall return error)
-    local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
-
-    if (numRemovedElements < 1) then return -3 end
-
-    -- Trim events before emiting them to avoid trimming events emitted in this script
-    trimEvents(KEYS[11], KEYS[4])
-
-    -- If job has a parent we need to
-    -- 1) remove this job id from parents dependencies
-    -- 2) move the job Id to parent "processed" set
-    -- 3) push the results into parent "results" list
-    -- 4) if parent's dependencies is empty, then move parent to "wait/paused". Note it may be a different queue!.
-    if parentId == "" and parentKey ~= "" then
-        parentId = getJobIdFromKey(parentKey)
-        parentQueueKey = getJobKeyPrefix(parentKey, ":" .. parentId)
-    end
-    if parentId ~= "" then
-        if ARGV[5] == "completed" then
-            local dependenciesSet = parentKey .. ":dependencies"
-            if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
-                updateParentDepsIfNeeded(parentKey, parentQueueKey, dependenciesSet,
-                                        parentId, jobIdKey, ARGV[4])
-            end
-        elseif opts['fpof'] then
-            moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
-        end
-    end
-
-    -- Remove job?
-    if maxCount ~= 0 then
-        local targetSet = KEYS[9]
-        -- Add to complete/failed set
-        rcall("ZADD", targetSet, timestamp, jobId)
-        rcall("HMSET", jobIdKey, ARGV[3], ARGV[4], "finishedOn", timestamp)
-        -- "returnvalue" / "failedReason" and "finishedOn"
-
-        -- Remove old jobs?
-        local prefix = ARGV[8]
-
-        if maxAge ~= nil then
-            removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
-        end
-
-        if maxCount ~= nil and maxCount > 0 then
-            removeJobsByMaxCount(maxCount, targetSet, prefix)
-        end
-    else
-        rcall("DEL", jobIdKey, jobIdKey .. ':logs', jobIdKey .. ':processed')
-    end
-
-    rcall("XADD", KEYS[4], "*", "event", ARGV[5], "jobId", jobId, ARGV[3],
-          ARGV[4])
-
-    if ARGV[5] == "failed" then
-        if tonumber(attemptsMade) >= tonumber(attempts) then
-            rcall("XADD", KEYS[4], "*", "event", "retries-exhausted", "jobId",
-                  jobId, "attemptsMade", attemptsMade)
-        end
-    end
-
-    -- Collect metrics
-    if maxMetricsSize ~= "" then
-        collectMetrics(KEYS[12], KEYS[12]..':data', maxMetricsSize, timestamp)
-    end
-
-    -- Try to get next job to avoid an extra roundtrip if the queue is not closing,
-    -- and not rate limited.
-    if (ARGV[7] == "1") then
-
-        -- Check if there are delayed jobs that can be promoted
-        promoteDelayedJobs(KEYS[7], KEYS[1], KEYS[3], KEYS[8], KEYS[11], KEYS[4], ARGV[8], timestamp)
-
-        jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
-
-        if jobId == "0" then
-            rcall("LREM", KEYS[2], 1, 0)
-        elseif jobId then
-            return moveJobFromWaitToActive(KEYS, ARGV[8], jobId, timestamp, opts)
-        end
-
-        -- Return the timestamp for the next delayed job if any.
-        local nextTimestamp = getNextDelayedTimestamp(KEYS[7])
-        if (nextTimestamp ~= nil) then
-            -- The result is guaranteed to be positive, since the
-            -- ZRANGEBYSCORE command would have return a job otherwise.
-            return nextTimestamp - timestamp
-        end
-    end
-
-    local waitLen = rcall("LLEN", KEYS[1])
-    if waitLen == 0 then
-        local activeLen = rcall("LLEN", KEYS[2])
-
-        if activeLen == 0 then
-            rcall("XADD", KEYS[4], "*", "event", "drained")
-        end
-    end  
-
-    return 0
-else
-    return -1
-end

package/dist/esm/commands/obliterate-2.lua

@@ -1,94 +0,0 @@
---[[
-  Completely obliterates a queue and all of its contents
-  
-  Input:
-    KEYS[1] meta
-    KEYS[2] base
-
-    ARGV[1] count
-    ARGV[2] force
-]]
-
--- This command completely destroys a queue including all of its jobs, current or past 
--- leaving no trace of its existence. Since this script needs to iterate to find all the job
--- keys, consider that this call may be slow for very large queues.
-
--- The queue needs to be "paused" or it will return an error
--- If the queue has currently active jobs then the script by default will return error,
--- however this behaviour can be overrided using the 'force' option.
-local maxCount = tonumber(ARGV[1])
-local baseKey = KEYS[2]
-
-local rcall = redis.call
-
---- @include "includes/removeJobs"
-
-local function removeLockKeys(keys)
-  for i, key in ipairs(keys) do
-    rcall("DEL", baseKey .. key .. ':lock')
-  end
-end
-
--- 1) Check if paused, if not return with error.
-if rcall("HEXISTS", KEYS[1], "paused") ~= 1 then
-  return -1 -- Error, NotPaused
-end
-
--- 2) Check if there are active jobs, if there are and not "force" return error.
-local activeKey = baseKey .. 'active'
-local activeJobs = getListItems(activeKey, maxCount)
-if (#activeJobs > 0) then
-  if(ARGV[2] == "") then 
-    return -2 -- Error, ExistActiveJobs
-  end
-end
-
-removeLockKeys(activeJobs)
-maxCount = removeJobs(activeJobs, true, baseKey, maxCount)
-rcall("LTRIM", activeKey, #activeJobs, -1)
-if(maxCount <= 0) then
-  return 1
-end
-
-local delayedKey = baseKey .. 'delayed'
-maxCount = removeZSetJobs(delayedKey, true, baseKey, maxCount)
-if(maxCount <= 0) then
-  return 1
-end
-
-local completedKey = baseKey .. 'completed'
-maxCount = removeZSetJobs(completedKey, true, baseKey, maxCount)
-if(maxCount <= 0) then
-  return 1
-end
-
-local waitKey = baseKey .. 'paused'
-maxCount = removeListJobs(waitKey, true, baseKey, maxCount)
-if(maxCount <= 0) then
-  return 1
-end
-
-local failedKey = baseKey .. 'failed'
-maxCount = removeZSetJobs(failedKey, true, baseKey, maxCount)
-if(maxCount <= 0) then
-  return 1
-end
-
-if(maxCount > 0) then
-  rcall("DEL", 
-    baseKey .. 'priority',
-    baseKey .. 'events',
-    baseKey .. 'delay', 
-    baseKey .. 'stalled-check',
-    baseKey .. 'stalled',
-    baseKey .. 'id',
-    baseKey .. 'meta',
-    baseKey .. 'repeat',
-    baseKey .. 'metrics:completed',
-    baseKey .. 'metrics:completed:data',
-    baseKey .. 'metrics:failed',
-    baseKey .. 'metrics:failed:data')
-  return 0
-else
-  return 1
-end

package/dist/esm/commands/removeJob-1.lua

@@ -1,72 +0,0 @@
---[[
-    Remove a job from all the queues it may be in as well as all its data.
-    In order to be able to remove a job, it cannot be active.
-
-    Input:
-      KEYS[1] queue prefix
-      ARGV[1] jobId
-
-    Events:
-      'removed'
-]]
-
-local rcall = redis.call
-
--- Includes
---- @include "includes/destructureJobKey"
---- @include "includes/isLocked"
---- @include "includes/removeJobFromAnyState"
---- @include "includes/removeParentDependencyKey"
-
-local function removeJob( prefix, jobId, parentKey)
-    local jobKey = prefix .. jobId;
-
-    removeParentDependencyKey(jobKey, false, parentKey)
-
-    -- Check if this job has children
-    -- If so, we are going to try to remove the children recursively in deep first way because
-    -- if some job is locked we must exit with and error.
-    --local countProcessed = rcall("HLEN", jobKey .. ":processed")
-    local processed = rcall("HGETALL", jobKey .. ":processed")
-
-    if (#processed > 0) then
-        for i = 1, #processed, 2 do
-            local childJobId = getJobIdFromKey(processed[i])
-            local childJobPrefix = getJobKeyPrefix(processed[i], childJobId)
-            removeJob( childJobPrefix, childJobId, jobKey )
-        end
-    end
-
-    local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
-    if (#dependencies > 0) then
-        for i, childJobKey in ipairs(dependencies) do
-            -- We need to get the jobId for this job.
-            local childJobId = getJobIdFromKey(childJobKey)
-            local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
-            removeJob( childJobPrefix, childJobId, jobKey )
-        end
-    end
-
-    local prev = removeJobFromAnyState(prefix, jobId)
-
-    rcall("ZREM", prefix .. "priority", jobId)
-    rcall("DEL", jobKey, jobKey .. ":logs", jobKey .. ":dependencies", jobKey .. ":processed")
-
-    -- -- delete keys related to rate limiter
-        -- local limiterIndexTable = KEYS[10] .. ":index"
-        -- local limitedSetKey = rcall("HGET", limiterIndexTable, jobId)
-        -- if limitedSetKey then
-        --     rcall("SREM", limitedSetKey, jobId)
-        --     rcall("HDEL", limiterIndexTable, jobId)
-    -- end
-
-    rcall("XADD", prefix .. "events", "*", "event", "removed", "jobId", jobId, "prev", prev);
-end
-
-local prefix = KEYS[1]
-
-if not isLocked(prefix, ARGV[1]) then
-    removeJob(prefix, ARGV[1])
-    return 1
-end
-return 0