@tachybase/plugin-adapter-bullmq 0.23.8

package/dist/node_modules/bullmq/dist/esm/commands/moveToFinished-14.lua

@@ -0,0 +1,283 @@
+--[[
+  Move job from active to a finished status (completed o failed)
+  A job can only be moved to completed if it was active.
+  The job must be locked before it can be moved to a finished status,
+  and the lock must be released in this script.
+
+    Input:
+      KEYS[1] wait key
+      KEYS[2] active key
+      KEYS[3] prioritized key
+      KEYS[4] event stream key
+      KEYS[5] stalled key
+
+      -- Rate limiting
+      KEYS[6] rate limiter key
+      KEYS[7] delayed key
+
+      KEYS[8] paused key
+      KEYS[9] meta key
+      KEYS[10] pc priority counter
+
+      KEYS[11] completed/failed key
+      KEYS[12] jobId key
+      KEYS[13] metrics key
+      KEYS[14] marker key
+
+      ARGV[1]  jobId
+      ARGV[2]  timestamp
+      ARGV[3]  msg property returnvalue / failedReason
+      ARGV[4]  return value / failed reason
+      ARGV[5]  target (completed/failed)
+      ARGV[6]  fetch next?
+      ARGV[7]  keys prefix
+      ARGV[8]  opts
+      ARGV[9]  job fields to update
+
+      opts - token - lock token
+      opts - keepJobs
+      opts - lockDuration - lock duration in milliseconds
+      opts - attempts max attempts
+      opts - maxMetricsSize
+      opts - fpof - fail parent on fail
+      opts - idof - ignore dependency on fail
+      opts - rdof - remove dependency on fail
+
+    Output:
+      0 OK
+      -1 Missing key.
+      -2 Missing lock.
+      -3 Job not in active set
+      -4 Job has pending dependencies
+      -6 Lock is not owned by this client
+
+    Events:
+      'completed/failed'
+]]
+local rcall = redis.call
+
+--- Includes
+--- @include "includes/collectMetrics"
+--- @include "includes/getNextDelayedTimestamp"
+--- @include "includes/getRateLimitTTL"
+--- @include "includes/getTargetQueueList"
+--- @include "includes/moveJobFromPriorityToActive"
+--- @include "includes/moveParentFromWaitingChildrenToFailed"
+--- @include "includes/moveParentToWaitIfNeeded"
+--- @include "includes/prepareJobForProcessing"
+--- @include "includes/promoteDelayedJobs"
+--- @include "includes/removeDeduplicationKeyIfNeeded"
+--- @include "includes/removeJobKeys"
+--- @include "includes/removeJobsByMaxAge"
+--- @include "includes/removeJobsByMaxCount"
+--- @include "includes/removeLock"
+--- @include "includes/removeParentDependencyKey"
+--- @include "includes/trimEvents"
+--- @include "includes/updateParentDepsIfNeeded"
+--- @include "includes/updateJobFields"
+
+local jobIdKey = KEYS[12]
+if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
+    local opts = cmsgpack.unpack(ARGV[8])
+
+    local token = opts['token']
+
+    local errorCode = removeLock(jobIdKey, KEYS[5], token, ARGV[1])
+    if errorCode < 0 then
+        return errorCode
+    end
+
+    updateJobFields(jobIdKey, ARGV[9]);
+
+    local attempts = opts['attempts']
+    local maxMetricsSize = opts['maxMetricsSize']
+    local maxCount = opts['keepJobs']['count']
+    local maxAge = opts['keepJobs']['age']
+
+    if rcall("SCARD", jobIdKey .. ":dependencies") ~= 0 then -- // Make sure it does not have pending dependencies
+        return -4
+    end
+
+    local jobAttributes = rcall("HMGET", jobIdKey, "parentKey", "parent", "deid")
+    local parentKey = jobAttributes[1] or ""
+    local parentId = ""
+    local parentQueueKey = ""
+    if jobAttributes[2] ~= false then
+        local jsonDecodedParent = cjson.decode(jobAttributes[2])
+        parentId = jsonDecodedParent['id']
+        parentQueueKey = jsonDecodedParent['queueKey']
+    end
+
+    local jobId = ARGV[1]
+    local timestamp = ARGV[2]
+
+    -- Remove from active list (if not active we shall return error)
+    local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
+
+    if (numRemovedElements < 1) then return -3 end
+
+    local eventStreamKey = KEYS[4]
+    local metaKey = KEYS[9]
+    -- Trim events before emiting them to avoid trimming events emitted in this script
+    trimEvents(metaKey, eventStreamKey)
+
+    local prefix = ARGV[7]
+
+    removeDeduplicationKeyIfNeeded(prefix, jobAttributes[3])
+
+    -- If job has a parent we need to
+    -- 1) remove this job id from parents dependencies
+    -- 2) move the job Id to parent "processed" set
+    -- 3) push the results into parent "results" list
+    -- 4) if parent's dependencies is empty, then move parent to "wait/paused". Note it may be a different queue!.
+    if parentId == "" and parentKey ~= "" then
+        parentId = getJobIdFromKey(parentKey)
+        parentQueueKey = getJobKeyPrefix(parentKey, ":" .. parentId)
+    end
+
+    if parentId ~= "" then
+        if ARGV[5] == "completed" then
+            local dependenciesSet = parentKey .. ":dependencies"
+            if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
+                updateParentDepsIfNeeded(parentKey, parentQueueKey,
+                                         dependenciesSet, parentId, jobIdKey,
+                                         ARGV[4], timestamp)
+            end
+        else
+            if opts['fpof'] then
+                moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey,
+                                                      parentId, jobIdKey,
+                                                      timestamp)
+            elseif opts['idof'] or opts['rdof'] then
+                local dependenciesSet = parentKey .. ":dependencies"
+                if rcall("SREM", dependenciesSet, jobIdKey) == 1 then
+                    moveParentToWaitIfNeeded(parentQueueKey, dependenciesSet,
+                                             parentKey, parentId, timestamp)
+                    if opts['idof'] then
+                        local failedSet = parentKey .. ":failed"
+                        rcall("HSET", failedSet, jobIdKey, ARGV[4])
+                    end
+                end
+            end
+        end
+    end
+
+    local attemptsMade = rcall("HINCRBY", jobIdKey, "atm", 1)
+
+    -- Remove job?
+    if maxCount ~= 0 then
+        local targetSet = KEYS[11]
+        -- Add to complete/failed set
+        rcall("ZADD", targetSet, timestamp, jobId)
+        rcall("HMSET", jobIdKey, ARGV[3], ARGV[4], "finishedOn", timestamp)
+        -- "returnvalue" / "failedReason" and "finishedOn"
+
+        -- Remove old jobs?
+        if maxAge ~= nil then
+            removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix)
+        end
+
+        if maxCount ~= nil and maxCount > 0 then
+            removeJobsByMaxCount(maxCount, targetSet, prefix)
+        end
+    else
+        removeJobKeys(jobIdKey)
+        if parentKey ~= "" then
+            -- TODO: when a child is removed when finished, result or failure in parent
+            -- must not be deleted, those value references should be deleted when the parent
+            -- is deleted
+            removeParentDependencyKey(jobIdKey, false, parentKey, jobAttributes[3])
+        end
+    end
+
+    rcall("XADD", eventStreamKey, "*", "event", ARGV[5], "jobId", jobId, ARGV[3],
+          ARGV[4])
+
+    if ARGV[5] == "failed" then
+        if tonumber(attemptsMade) >= tonumber(attempts) then
+            rcall("XADD", eventStreamKey, "*", "event", "retries-exhausted", "jobId",
+                  jobId, "attemptsMade", attemptsMade)
+        end
+    end
+
+    -- Collect metrics
+    if maxMetricsSize ~= "" then
+        collectMetrics(KEYS[13], KEYS[13] .. ':data', maxMetricsSize, timestamp)
+    end
+
+    -- Try to get next job to avoid an extra roundtrip if the queue is not closing,
+    -- and not rate limited.
+    if (ARGV[6] == "1") then
+
+        local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[2], KEYS[1], KEYS[8])
+
+        local markerKey = KEYS[14]
+        -- Check if there are delayed jobs that can be promoted
+        promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[3], eventStreamKey, prefix,
+                           timestamp, KEYS[10], isPausedOrMaxed)
+
+        local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
+        -- Check if we are rate limited first.
+        local expireTime = getRateLimitTTL(maxJobs, KEYS[6])
+
+        if expireTime > 0 then return {0, 0, expireTime, 0} end
+
+        -- paused or maxed queue
+        if isPausedOrMaxed then return {0, 0, 0, 0} end
+
+        jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
+
+        if jobId then
+            -- Markers in waitlist DEPRECATED in v5: Remove in v6.
+            if string.sub(jobId, 1, 2) == "0:" then
+                rcall("LREM", KEYS[2], 1, jobId)
+
+                -- If jobId is special ID 0:delay (delay greater than 0), then there is no job to process
+                -- but if ID is 0:0, then there is at least 1 prioritized job to process
+                if jobId == "0:0" then
+                    jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2],
+                                                        KEYS[10])
+                    return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId,
+                                                   timestamp, maxJobs, markerKey,
+                                                   opts)
+                end
+            else
+                return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId,
+                                               timestamp, maxJobs, markerKey,
+                                               opts)
+            end
+        else
+            jobId = moveJobFromPriorityToActive(KEYS[3], KEYS[2], KEYS[10])
+            if jobId then
+                return prepareJobForProcessing(prefix, KEYS[6], eventStreamKey, jobId,
+                                               timestamp, maxJobs, markerKey,
+                                               opts)
+            end
+        end
+
+        -- Return the timestamp for the next delayed job if any.
+        local nextTimestamp = getNextDelayedTimestamp(KEYS[7])
+        if nextTimestamp ~= nil then
+            -- The result is guaranteed to be positive, since the
+            -- ZRANGEBYSCORE command would have return a job otherwise.
+            return {0, 0, 0, nextTimestamp}
+        end
+    end
+
+    local waitLen = rcall("LLEN", KEYS[1])
+    if waitLen == 0 then
+        local activeLen = rcall("LLEN", KEYS[2])
+
+        if activeLen == 0 then
+            local prioritizedLen = rcall("ZCARD", KEYS[3])
+
+            if prioritizedLen == 0 then
+                rcall("XADD", eventStreamKey, "*", "event", "drained")
+            end
+        end
+    end
+
+    return 0
+else
+    return -1
+end

package/dist/node_modules/bullmq/dist/esm/commands/moveToWaitingChildren-5.lua

@@ -0,0 +1,68 @@
+--[[
+  Moves job from active to waiting children set.
+
+  Input:
+    KEYS[1] lock key
+    KEYS[2] active key
+    KEYS[3] waitChildrenKey key
+    KEYS[4] job key
+    KEYS[5] stalled key
+
+    ARGV[1] token
+    ARGV[2] child key
+    ARGV[3] timestamp
+    ARGV[4] the id of the job
+
+  Output:
+    0 - OK
+    1 - There are not pending dependencies.
+   -1 - Missing job.
+   -2 - Missing lock
+   -3 - Job not in active set
+]]
+local rcall = redis.call
+local stalledKey = KEYS[5]
+
+--- Includes
+--- @include "includes/removeLock"
+
+local function moveToWaitingChildren (activeKey, waitingChildrenKey, jobId,
+    timestamp)
+  local score = tonumber(timestamp)
+
+  local numRemovedElements = rcall("LREM", activeKey, -1, jobId)
+
+  if(numRemovedElements < 1) then
+    return -3
+  end
+
+  rcall("ZADD", waitingChildrenKey, score, jobId)
+
+  return 0
+end
+
+if rcall("EXISTS", KEYS[4]) == 1 then
+  if ARGV[2] ~= "" then
+    if rcall("SISMEMBER", KEYS[4] .. ":dependencies", ARGV[2]) ~= 0 then
+      local errorCode = removeLock(KEYS[4], stalledKey, ARGV[1], ARGV[4])
+      if errorCode < 0 then
+        return errorCode
+      end
+      return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3])
+    end
+
+    return 1
+  else
+    if rcall("SCARD", KEYS[4] .. ":dependencies") ~= 0 then 
+      local errorCode = removeLock(KEYS[4], stalledKey, ARGV[1], ARGV[4])
+      if errorCode < 0 then
+        return errorCode
+      end
+      return moveToWaitingChildren(KEYS[2], KEYS[3], ARGV[4], ARGV[3])
+    end
+
+    return 1
+  end
+end
+
+return -1

package/dist/node_modules/bullmq/dist/esm/commands/obliterate-2.lua

@@ -0,0 +1,118 @@
+--[[
+  Completely obliterates a queue and all of its contents
+  This command completely destroys a queue including all of its jobs, current or past 
+  leaving no trace of its existence. Since this script needs to iterate to find all the job
+  keys, consider that this call may be slow for very large queues.
+
+  The queue needs to be "paused" or it will return an error
+  If the queue has currently active jobs then the script by default will return error,
+  however this behaviour can be overrided using the 'force' option.
+  
+  Input:
+    KEYS[1] meta
+    KEYS[2] base
+
+    ARGV[1] count
+    ARGV[2] force
+]]
+
+local maxCount = tonumber(ARGV[1])
+local baseKey = KEYS[2]
+
+local rcall = redis.call
+
+-- Includes
+--- @include "includes/removeJobs"
+--- @include "includes/removeListJobs"
+--- @include "includes/removeZSetJobs"
+
+local function removeLockKeys(keys)
+  for i, key in ipairs(keys) do
+    rcall("DEL", baseKey .. key .. ':lock')
+  end
+end
+
+-- 1) Check if paused, if not return with error.
+if rcall("HEXISTS", KEYS[1], "paused") ~= 1 then
+  return -1 -- Error, NotPaused
+end
+
+-- 2) Check if there are active jobs, if there are and not "force" return error.
+local activeKey = baseKey .. 'active'
+local activeJobs = getListItems(activeKey, maxCount)
+if (#activeJobs > 0) then
+  if(ARGV[2] == "") then 
+    return -2 -- Error, ExistActiveJobs
+  end
+end
+
+removeLockKeys(activeJobs)
+maxCount = removeJobs(activeJobs, true, baseKey, maxCount)
+rcall("LTRIM", activeKey, #activeJobs, -1)
+if(maxCount <= 0) then
+  return 1
+end
+
+local delayedKey = baseKey .. 'delayed'
+maxCount = removeZSetJobs(delayedKey, true, baseKey, maxCount)
+if(maxCount <= 0) then
+  return 1
+end
+
+local repeatKey = baseKey .. 'repeat'
+local repeatJobsIds = getZSetItems(repeatKey, maxCount)
+for i, key in ipairs(repeatJobsIds) do
+  local jobKey = repeatKey .. ":" .. key
+  rcall("DEL", jobKey)
+end
+if(#repeatJobsIds > 0) then
+  for from, to in batches(#repeatJobsIds, 7000) do
+    rcall("ZREM", repeatKey, unpack(repeatJobsIds, from, to))
+  end
+end
+maxCount = maxCount - #repeatJobsIds
+if(maxCount <= 0) then
+  return 1
+end
+
+local completedKey = baseKey .. 'completed'
+maxCount = removeZSetJobs(completedKey, true, baseKey, maxCount)
+if(maxCount <= 0) then
+  return 1
+end
+
+local waitKey = baseKey .. 'paused'
+maxCount = removeListJobs(waitKey, true, baseKey, maxCount)
+if(maxCount <= 0) then
+  return 1
+end
+
+local prioritizedKey = baseKey .. 'prioritized'
+maxCount = removeZSetJobs(prioritizedKey, true, baseKey, maxCount)
+if(maxCount <= 0) then
+  return 1
+end
+
+local failedKey = baseKey .. 'failed'
+maxCount = removeZSetJobs(failedKey, true, baseKey, maxCount)
+if(maxCount <= 0) then
+  return 1
+end
+
+if(maxCount > 0) then
+  rcall("DEL",
+    baseKey .. 'events',
+    baseKey .. 'delay', 
+    baseKey .. 'stalled-check',
+    baseKey .. 'stalled',
+    baseKey .. 'id',
+    baseKey .. 'pc',
+    baseKey .. 'meta',
+    baseKey .. 'metrics:completed',
+    baseKey .. 'metrics:completed:data',
+    baseKey .. 'metrics:failed',
+    baseKey .. 'metrics:failed:data')
+  return 0
+else
+  return 1
+end

package/dist/node_modules/bullmq/dist/esm/commands/paginate-1.lua

@@ -0,0 +1,49 @@
+--[[
+    Paginate a set or hash
+
+    Input:
+      KEYS[1] key pointing to the set or hash to be paginated.
+    
+      ARGV[1]  page start offset
+      ARGV[2]  page end offset (-1 for all the elements)
+      ARGV[3]  cursor
+      ARGV[4]  offset
+      ARGV[5]  max iterations
+      ARGV[6]  fetch jobs?
+      
+    Output:
+      [cursor, offset, items, numItems]
+]]
+local rcall = redis.call
+
+-- Includes
+--- @include "includes/findPage"
+
+local key = KEYS[1]
+local scanCommand = "SSCAN"
+local countCommand = "SCARD"
+local type = rcall("TYPE", key)["ok"]
+
+if type == "none" then
+    return {0, 0, {}, 0}
+elseif type == "hash" then
+    scanCommand = "HSCAN"
+    countCommand = "HLEN"
+elseif type ~= "set" then
+    return
+        redis.error_reply("Pagination is only supported for sets and hashes.")
+end
+
+local numItems = rcall(countCommand, key)
+local startOffset = tonumber(ARGV[1])
+local endOffset = tonumber(ARGV[2])
+if endOffset == -1 then 
+  endOffset = numItems
+end
+local pageSize = (endOffset - startOffset) + 1
+
+local cursor, offset, items, jobs = findPage(key, scanCommand, startOffset,
+                                       pageSize, ARGV[3], tonumber(ARGV[4]),
+                                       tonumber(ARGV[5]), ARGV[6])
+
+return {cursor, offset, items, numItems, jobs}

package/dist/node_modules/bullmq/dist/esm/commands/pause-7.lua

@@ -0,0 +1,42 @@
+--[[
+  Pauses or resumes a queue globably.
+
+  Input:
+    KEYS[1] 'wait' or 'paused''
+    KEYS[2] 'paused' or 'wait'
+    KEYS[3] 'meta'
+    KEYS[4] 'prioritized'
+    KEYS[5] events stream key
+    KEYS[6] 'delayed'
+    KEYS|7] 'marker'
+
+    ARGV[1] 'paused' or 'resumed'
+
+  Event:
+    publish paused or resumed event.
+]]
+local rcall = redis.call
+
+-- Includes
+--- @include "includes/addDelayMarkerIfNeeded"
+
+local markerKey = KEYS[7]
+local hasJobs = rcall("EXISTS", KEYS[1]) == 1
+--TODO: check this logic to be reused when changing a delay
+if hasJobs then rcall("RENAME", KEYS[1], KEYS[2]) end
+
+if ARGV[1] == "paused" then
+    rcall("HSET", KEYS[3], "paused", 1)
+    rcall("DEL", markerKey)
+else
+    rcall("HDEL", KEYS[3], "paused")
+
+    if hasJobs or rcall("ZCARD", KEYS[4]) > 0 then
+        -- Add marker if there are waiting or priority jobs
+        rcall("ZADD", markerKey, 0, "0")
+    else
+        addDelayMarkerIfNeeded(markerKey, KEYS[6])
+    end
+end
+
+rcall("XADD", KEYS[5], "*", "event", ARGV[1]);

package/dist/node_modules/bullmq/dist/esm/commands/promote-9.lua

@@ -0,0 +1,62 @@
+--[[
+  Promotes a job that is currently "delayed" to the "waiting" state
+
+    Input:
+      KEYS[1] 'delayed'
+      KEYS[2] 'wait'
+      KEYS[3] 'paused'
+      KEYS[4] 'meta'
+      KEYS[5] 'prioritized'
+      KEYS[6] 'active'
+      KEYS[7] 'pc' priority counter
+      KEYS[8] 'event stream'
+      KEYS[9] 'marker'
+
+      ARGV[1]  queue.toKey('')
+      ARGV[2]  jobId
+
+    Output:
+       0 - OK
+      -3 - Job not in delayed zset.
+
+    Events:
+      'waiting'
+]]
+local rcall = redis.call
+local jobId = ARGV[2]
+
+-- Includes
+--- @include "includes/addJobInTargetList"
+--- @include "includes/addJobWithPriority"
+--- @include "includes/getTargetQueueList"
+
+if rcall("ZREM", KEYS[1], jobId) == 1 then
+    local jobKey = ARGV[1] .. jobId
+    local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
+    local metaKey = KEYS[4]
+    local markerKey = KEYS[9]
+
+    -- Remove delayed "marker" from the wait list if there is any.
+    -- Since we are adding a job we do not need the marker anymore.
+    -- Markers in waitlist DEPRECATED in v5: Remove in v6.
+    local target, isPausedOrMaxed = getTargetQueueList(metaKey, KEYS[6], KEYS[2], KEYS[3])
+    local marker = rcall("LINDEX", target, 0)
+    if marker and string.sub(marker, 1, 2) == "0:" then rcall("LPOP", target) end
+
+    if priority == 0 then
+        -- LIFO or FIFO
+        addJobInTargetList(target, markerKey, "LPUSH", isPausedOrMaxed, jobId)
+    else
+        addJobWithPriority(markerKey, KEYS[5], priority, jobId, KEYS[7], isPausedOrMaxed)
+    end
+
+    -- Emit waiting event (wait..ing@token)
+    rcall("XADD", KEYS[8], "*", "event", "waiting", "jobId", jobId, "prev",
+          "delayed");
+
+    rcall("HSET", jobKey, "delay", 0)
+
+    return 0
+else
+    return -3
+end

package/dist/node_modules/bullmq/dist/esm/commands/releaseLock-1.lua

@@ -0,0 +1,19 @@
+--[[
+  Release lock
+
+    Input:
+      KEYS[1] 'lock',
+    
+      ARGV[1]  token
+      ARGV[2]  lock duration in milliseconds
+      
+    Output:
+      "OK" if lock extented succesfully.
+]]
+local rcall = redis.call
+
+if rcall("GET", KEYS[1]) == ARGV[1] then
+  return rcall("DEL", KEYS[1])
+else
+  return 0
+end

package/dist/node_modules/bullmq/dist/esm/commands/removeChildDependency-1.lua

@@ -0,0 +1,34 @@
+--[[
+  Break parent-child dependency by removing
+  child reference from parent
+
+  Input:
+    KEYS[1] 'key' prefix,
+
+    ARGV[1] job key
+    ARGV[2] parent key
+
+    Output:
+       0  - OK
+       1  - There is not relationship.
+      -1  - Missing job key
+      -5  - Missing parent key
+]]
+local rcall = redis.call
+local jobKey = ARGV[1]
+local parentKey = ARGV[2]
+
+-- Includes
+--- @include "includes/removeParentDependencyKey"
+
+if rcall("EXISTS", jobKey) ~= 1 then return -1 end
+
+if rcall("EXISTS", parentKey) ~= 1 then return -5 end
+
+if removeParentDependencyKey(jobKey, false, parentKey, KEYS[1], nil) then
+  rcall("HDEL", jobKey, "parentKey", "parent")
+
+  return 0
+else
+  return 1
+end