@tachybase/plugin-adapter-bullmq 0.23.8

package/dist/node_modules/bullmq/dist/esm/commands/includes/prepareJobForProcessing.lua

@@ -0,0 +1,50 @@
+
+--[[
+  Function to move job from wait state to active.
+  Input:
+    opts - token - lock token
+    opts - lockDuration
+    opts - limiter
+]]
+
+-- Includes
+--- @include "addBaseMarkerIfNeeded"
+
+local function prepareJobForProcessing(keyPrefix, rateLimiterKey, eventStreamKey,
+    jobId, processedOn, maxJobs, markerKey, opts)
+  local jobKey = keyPrefix .. jobId
+
+  -- Check if we need to perform rate limiting.
+  if maxJobs then
+    local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
+
+    if jobCounter == 1 then
+      local limiterDuration = opts['limiter'] and opts['limiter']['duration']
+      local integerDuration = math.floor(math.abs(limiterDuration))
+      rcall("PEXPIRE", rateLimiterKey, integerDuration)
+    end
+  end
+
+  local lockKey = jobKey .. ':lock'
+
+  -- get a lock
+  if opts['token'] ~= "0" then
+    rcall("SET", lockKey, opts['token'], "PX", opts['lockDuration'])
+  end
+
+  local optionalValues = {}
+
+  if opts['name'] then
+    -- Set "processedBy" field to the worker name
+    table.insert(optionalValues, "pb")
+    table.insert(optionalValues, opts['name'])
+  end
+
+  rcall("XADD", eventStreamKey, "*", "event", "active", "jobId", jobId, "prev", "waiting")
+  rcall("HMSET", jobKey, "processedOn", processedOn, unpack(optionalValues))
+  rcall("HINCRBY", jobKey, "ats", 1)
+
+  addBaseMarkerIfNeeded(markerKey, false)
+
+  return {rcall("HGETALL", jobKey), jobId, 0, 0} -- get job data
+end

package/dist/node_modules/bullmq/dist/esm/commands/includes/promoteDelayedJobs.lua

@@ -0,0 +1,40 @@
+--[[
+  Updates the delay set, by moving delayed jobs that should
+  be processed now to "wait".
+
+     Events:
+      'waiting'
+]]
+
+-- Includes
+--- @include "addJobInTargetList"
+--- @include "addJobWithPriority"
+
+-- Try to get as much as 1000 jobs at once
+local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedKey,
+                                  eventStreamKey, prefix, timestamp, priorityCounterKey, isPaused)
+    local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000 - 1, "LIMIT", 0, 1000)
+
+    if (#jobs > 0) then
+        rcall("ZREM", delayedKey, unpack(jobs))
+
+        for _, jobId in ipairs(jobs) do
+            local jobKey = prefix .. jobId
+            local priority =
+                tonumber(rcall("HGET", jobKey, "priority")) or 0
+
+            if priority == 0 then
+                -- LIFO or FIFO
+                addJobInTargetList(targetKey, markerKey, "LPUSH", isPaused, jobId)
+            else
+                addJobWithPriority(markerKey, prioritizedKey, priority,
+                  jobId, priorityCounterKey, isPaused)
+            end
+
+            -- Emit waiting event
+            rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
+                  jobId, "prev", "delayed")
+            rcall("HSET", jobKey, "delay", 0)
+        end
+    end
+end

package/dist/node_modules/bullmq/dist/esm/commands/includes/pushBackJobWithPriority.lua

@@ -0,0 +1,9 @@
+--[[
+  Function to push back job considering priority in front of same prioritized jobs.
+]]
+local function pushBackJobWithPriority(prioritizedKey, priority, jobId)
+  -- in order to put it at front of same prioritized jobs
+  -- we consider prioritized counter as 0
+  local score = priority * 0x100000000
+  rcall("ZADD", prioritizedKey, score, jobId)
+end

package/dist/node_modules/bullmq/dist/esm/commands/includes/removeDeduplicationKey.lua

@@ -0,0 +1,11 @@
+--[[
+  Function to remove deduplication key.
+]]
+
+local function removeDeduplicationKey(prefixKey, jobKey)
+  local deduplicationId = rcall("HGET", jobKey, "deid")
+  if deduplicationId then
+    local deduplicationKey = prefixKey .. "de:" .. deduplicationId
+    rcall("DEL", deduplicationKey)
+  end
+end

package/dist/node_modules/bullmq/dist/esm/commands/includes/removeDeduplicationKeyIfNeeded.lua

@@ -0,0 +1,14 @@
+--[[
+  Function to remove deduplication key if needed.
+]]
+
+local function removeDeduplicationKeyIfNeeded(prefixKey, deduplicationId)
+  if deduplicationId then
+    local deduplicationKey = prefixKey .. "de:" .. deduplicationId
+    local pttl = rcall("PTTL", deduplicationKey)
+
+    if pttl == 0 or pttl == -1 then
+      rcall("DEL", deduplicationKey)
+    end
+  end
+end

package/dist/node_modules/bullmq/dist/esm/commands/includes/removeJob.lua

@@ -0,0 +1,17 @@
+--[[
+  Function to remove job.
+]]
+
+-- Includes
+--- @include "removeDeduplicationKey"
+--- @include "removeJobKeys"
+--- @include "removeParentDependencyKey"
+
+local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
+  local jobKey = baseKey .. jobId
+  removeParentDependencyKey(jobKey, hard, nil, baseKey)
+  if shouldRemoveDeduplicationKey then
+    removeDeduplicationKey(baseKey, jobKey)
+  end
+  removeJobKeys(jobKey)
+end

package/dist/node_modules/bullmq/dist/esm/commands/includes/removeJobFromAnyState.lua

@@ -0,0 +1,35 @@
+--[[
+  Function to remove from any state.
+
+  returns:
+    prev state
+]]
+
+local function removeJobFromAnyState( prefix, jobId)
+  -- We start with the ZSCORE checks, since they have O(1) complexity
+  if rcall("ZSCORE", prefix .. "completed", jobId) then
+    rcall("ZREM", prefix .. "completed", jobId)
+    return "completed"
+  elseif rcall("ZSCORE", prefix .. "waiting-children", jobId) then
+    rcall("ZREM", prefix .. "waiting-children", jobId)
+    return "waiting-children"
+  elseif rcall("ZSCORE", prefix .. "delayed", jobId) then
+    rcall("ZREM", prefix .. "delayed", jobId)
+    return "delayed"
+  elseif rcall("ZSCORE", prefix .. "failed", jobId) then
+    rcall("ZREM", prefix .. "failed", jobId)
+    return "failed"
+  elseif rcall("ZSCORE", prefix .. "prioritized", jobId) then
+    rcall("ZREM", prefix .. "prioritized", jobId)
+    return "prioritized"
+  -- We remove only 1 element from the list, since we assume they are not added multiple times
+  elseif rcall("LREM", prefix .. "wait", 1, jobId) == 1 then
+    return "wait"
+  elseif rcall("LREM", prefix .. "paused", 1, jobId) == 1 then
+    return "paused"
+  elseif rcall("LREM", prefix .. "active", 1, jobId) == 1 then
+    return "active"
+  end
+
+  return "unknown"
+end

package/dist/node_modules/bullmq/dist/esm/commands/includes/removeJobKeys.lua

@@ -0,0 +1,8 @@
+--[[
+  Function to remove job keys.
+]]
+
+local function removeJobKeys(jobKey)
+  return rcall("DEL", jobKey, jobKey .. ':logs',
+    jobKey .. ':dependencies', jobKey .. ':processed', jobKey .. ':failed')
+end

package/dist/node_modules/bullmq/dist/esm/commands/includes/removeJobs.lua

@@ -0,0 +1,13 @@
+--[[
+  Functions to remove jobs.
+]]
+
+-- Includes
+--- @include "removeJob"
+
+local function removeJobs(keys, hard, baseKey, max)
+  for i, key in ipairs(keys) do
+    removeJob(key, hard, baseKey, true --[[remove debounce key]])
+  end
+  return max - #keys
+end

package/dist/node_modules/bullmq/dist/esm/commands/includes/removeJobsByMaxAge.lua

@@ -0,0 +1,16 @@
+--[[
+  Functions to remove jobs by max age.
+]]
+
+-- Includes
+--- @include "removeJob"
+
+local function removeJobsByMaxAge(timestamp, maxAge, targetSet, prefix,
+  shouldRemoveDebounceKey)
+  local start = timestamp - maxAge * 1000
+  local jobIds = rcall("ZREVRANGEBYSCORE", targetSet, start, "-inf")
+  for i, jobId in ipairs(jobIds) do
+    removeJob(jobId, false, prefix, false --[[remove debounce key]])
+  end
+  rcall("ZREMRANGEBYSCORE", targetSet, "-inf", start)
+end

package/dist/node_modules/bullmq/dist/esm/commands/includes/removeJobsByMaxCount.lua

@@ -0,0 +1,15 @@
+--[[
+  Functions to remove jobs by max count.
+]]
+
+-- Includes
+--- @include "removeJob"
+
+local function removeJobsByMaxCount(maxCount, targetSet, prefix)
+  local start = maxCount
+  local jobIds = rcall("ZREVRANGE", targetSet, start, -1)
+  for i, jobId in ipairs(jobIds) do
+    removeJob(jobId, false, prefix, false --[[remove debounce key]])
+  end
+  rcall("ZREMRANGEBYRANK", targetSet, 0, -(maxCount + 1))
+end

package/dist/node_modules/bullmq/dist/esm/commands/includes/removeJobsOnFail.lua

@@ -0,0 +1,36 @@
+--[[
+  Functions to remove jobs when removeOnFail option is provided.
+]]
+
+-- Includes
+--- @include "removeJob"
+--- @include "removeJobsByMaxAge"
+--- @include "removeJobsByMaxCount"
+
+local function removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestamp)
+  local removeOnFailType = type(opts["removeOnFail"])
+  if removeOnFailType == "number" then
+    removeJobsByMaxCount(opts["removeOnFail"],
+                        failedKey, queueKeyPrefix)
+  elseif removeOnFailType == "boolean" then
+    if opts["removeOnFail"] then
+      removeJob(jobId, false, queueKeyPrefix,
+                false --[[remove debounce key]])
+      rcall("ZREM", failedKey, jobId)
+    end
+  elseif removeOnFailType ~= "nil" then
+    local maxAge = opts["removeOnFail"]["age"]
+    local maxCount = opts["removeOnFail"]["count"]
+
+    if maxAge ~= nil then
+      removeJobsByMaxAge(timestamp, maxAge,
+                        failedKey, queueKeyPrefix)
+    end
+
+    if maxCount ~= nil and maxCount > 0 then
+      removeJobsByMaxCount(maxCount, failedKey,
+                            queueKeyPrefix)
+    end
+  end 
+end
+  

package/dist/node_modules/bullmq/dist/esm/commands/includes/removeListJobs.lua

@@ -0,0 +1,17 @@
+--[[
+  Functions to remove jobs.
+]]
+
+-- Includes
+--- @include "removeJobs"
+
+local function getListItems(keyName, max)
+  return rcall('LRANGE', keyName, 0, max - 1)
+end
+
+local function removeListJobs(keyName, hard, baseKey, max)
+  local jobs = getListItems(keyName, max)
+  local count = removeJobs(jobs, hard, baseKey, max)
+  rcall("LTRIM", keyName, #jobs, -1)
+  return count
+end

package/dist/node_modules/bullmq/dist/esm/commands/includes/removeLock.lua

@@ -0,0 +1,19 @@
+local function removeLock(jobKey, stalledKey, token, jobId)
+  if token ~= "0" then
+    local lockKey = jobKey .. ':lock'
+    local lockToken = rcall("GET", lockKey)
+    if lockToken == token then
+      rcall("DEL", lockKey)
+      rcall("SREM", stalledKey, jobId)
+    else
+      if lockToken then
+        -- Lock exists but token does not match
+        return -6
+      else
+        -- Lock is missing completely
+        return -2
+      end
+    end
+  end
+  return 0
+end

package/dist/node_modules/bullmq/dist/esm/commands/includes/removeParentDependencyKey.lua

@@ -0,0 +1,90 @@
+--[[
+  Check if this job has a parent. If so we will just remove it from
+  the parent child list, but if it is the last child we should move the parent to "wait/paused"
+  which requires code from "moveToFinished"
+]]
+
+-- Includes
+--- @include "addJobInTargetList"
+--- @include "destructureJobKey"
+--- @include "getTargetQueueList"
+--- @include "removeJobKeys"
+
+local function moveParentToWait(parentPrefix, parentId, emitEvent)
+  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
+    parentPrefix .. "wait", parentPrefix .. "paused")
+  addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
+
+  if emitEvent then
+    local parentEventStream = parentPrefix .. "events"
+    rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
+  end
+end
+
+local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debounceId)
+  if parentKey then
+    local parentDependenciesKey = parentKey .. ":dependencies"
+    local result = rcall("SREM", parentDependenciesKey, jobKey)
+    if result > 0 then
+      local pendingDependencies = rcall("SCARD", parentDependenciesKey)
+      if pendingDependencies == 0 then
+        local parentId = getJobIdFromKey(parentKey)
+        local parentPrefix = getJobKeyPrefix(parentKey, parentId)
+
+        local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
+
+        if numRemovedElements == 1 then
+          if hard then -- remove parent in same queue
+            if parentPrefix == baseKey then
+              removeParentDependencyKey(parentKey, hard, nil, baseKey, nil)
+              removeJobKeys(parentKey)
+              if debounceId then
+                rcall("DEL", parentPrefix .. "de:" .. debounceId)
+              end
+            else
+              moveParentToWait(parentPrefix, parentId)
+            end
+          else
+            moveParentToWait(parentPrefix, parentId, true)
+          end
+        end
+      end
+      return true
+    end
+  else
+    local parentAttributes = rcall("HMGET", jobKey, "parentKey", "deid")
+    local missedParentKey = parentAttributes[1]
+    if( (type(missedParentKey) == "string") and missedParentKey ~= ""
+      and (rcall("EXISTS", missedParentKey) == 1)) then
+      local parentDependenciesKey = missedParentKey .. ":dependencies"
+      local result = rcall("SREM", parentDependenciesKey, jobKey)
+      if result > 0 then
+        local pendingDependencies = rcall("SCARD", parentDependenciesKey)
+        if pendingDependencies == 0 then
+          local parentId = getJobIdFromKey(missedParentKey)
+          local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
+
+          local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
+
+          if numRemovedElements == 1 then
+            if hard then
+              if parentPrefix == baseKey then
+                removeParentDependencyKey(missedParentKey, hard, nil, baseKey, nil)
+                removeJobKeys(missedParentKey)
+                if parentAttributes[2] then
+                  rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
+                end
+              else
+                moveParentToWait(parentPrefix, parentId)
+              end
+            else
+              moveParentToWait(parentPrefix, parentId, true)
+            end
+          end
+        end
+        return true
+      end
+    end
+  end
+  return false
+end

package/dist/node_modules/bullmq/dist/esm/commands/includes/removeZSetJobs.lua

@@ -0,0 +1,27 @@
+-- Includes
+--- @include "batches"
+--- @include "getZSetItems"
+--- @include "removeJobs"  
+
+local function removeZSetJobs(keyName, hard, baseKey, max, jobsToIgnore)
+  local jobs = getZSetItems(keyName, max)
+
+  -- filter out jobs to ignore
+  if jobsToIgnore then
+    local filteredJobs = {}
+    for i = 1, #jobs do
+      if not jobsToIgnore[jobs[i]] then
+        table.insert(filteredJobs, jobs[i])
+      end
+    end
+    jobs = filteredJobs
+  end
+
+  local count = removeJobs(jobs, hard, baseKey, max)
+  if(#jobs > 0) then
+    for from, to in batches(#jobs, 7000) do
+      rcall("ZREM", keyName, unpack(jobs, from, to))
+    end
+  end
+  return count
+end

package/dist/node_modules/bullmq/dist/esm/commands/includes/storeJob.lua

@@ -0,0 +1,36 @@
+--[[
+  Function to store a job
+]]
+local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
+                        parentKey, parentData, repeatJobKey)
+    local jsonOpts = cjson.encode(opts)
+    local delay = opts['delay'] or 0
+    local priority = opts['priority'] or 0
+    local debounceId = opts['de'] and opts['de']['id']
+    
+    local optionalValues = {}
+    if parentKey ~= nil then
+        table.insert(optionalValues, "parentKey")
+        table.insert(optionalValues, parentKey)
+        table.insert(optionalValues, "parent")
+        table.insert(optionalValues, parentData)
+    end
+
+    if repeatJobKey ~= nil then
+        table.insert(optionalValues, "rjk")
+        table.insert(optionalValues, repeatJobKey)
+    end
+
+    if debounceId then
+        table.insert(optionalValues, "deid")
+        table.insert(optionalValues, debounceId)
+    end
+
+    rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
+          "timestamp", timestamp, "delay", delay, "priority", priority,
+          unpack(optionalValues))
+
+    rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
+
+    return delay, priority
+end

package/dist/node_modules/bullmq/dist/esm/commands/includes/trimEvents.lua

@@ -0,0 +1,15 @@
+--[[
+  Function to trim events, default 10000.
+]]
+
+-- Includes
+--- @include "getOrSetMaxEvents"
+
+local function trimEvents(metaKey, eventStreamKey)
+  local maxEvents = getOrSetMaxEvents(metaKey)
+  if maxEvents ~= false then
+    rcall("XTRIM", eventStreamKey, "MAXLEN", "~", maxEvents)
+  else
+    rcall("XTRIM", eventStreamKey, "MAXLEN", "~", 10000)
+  end
+end

package/dist/node_modules/bullmq/dist/esm/commands/includes/updateExistingJobsParent.lua

@@ -0,0 +1,27 @@
+--[[
+    This function is used to update the parent's dependencies if the job
+    is already completed and about to be ignored. The parent must get its
+    dependencies updated to avoid the parent job being stuck forever in 
+    the waiting-children state.
+]]
+
+-- Includes
+--- @include "updateParentDepsIfNeeded"
+
+local function updateExistingJobsParent(parentKey, parent, parentData,
+                                        parentDependenciesKey, completedKey,
+                                        jobIdKey, jobId, timestamp)
+    if parentKey ~= nil then
+        if rcall("ZSCORE", completedKey, jobId) ~= false then
+            local returnvalue = rcall("HGET", jobIdKey, "returnvalue")
+            updateParentDepsIfNeeded(parentKey, parent['queueKey'],
+                                     parentDependenciesKey, parent['id'],
+                                     jobIdKey, returnvalue, timestamp)
+        else
+            if parentDependenciesKey ~= nil then
+                rcall("SADD", parentDependenciesKey, jobIdKey)
+            end
+        end
+        rcall("HMSET", jobIdKey, "parentKey", parentKey, "parent", parentData)
+    end
+end

package/dist/node_modules/bullmq/dist/esm/commands/includes/updateJobFields.lua

@@ -0,0 +1,11 @@
+--[[
+  Function to update a bunch of fields in a job.
+]]
+local function updateJobFields(jobKey, msgpackedFields)
+    if msgpackedFields and #msgpackedFields > 0 then
+        local fieldsToUpdate = cmsgpack.unpack(msgpackedFields)
+        if fieldsToUpdate then
+            redis.call("HMSET", jobKey, unpack(fieldsToUpdate))
+        end
+    end
+end

package/dist/node_modules/bullmq/dist/esm/commands/includes/updateParentDepsIfNeeded.lua

@@ -0,0 +1,13 @@
+--[[
+  Validate and move or add dependencies to parent.
+]]
+
+-- Includes
+--- @include "moveParentToWaitIfNeeded"
+
+local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
+  parentId, jobIdKey, returnvalue, timestamp )
+  local processedSet = parentKey .. ":processed"
+  rcall("HSET", processedSet, jobIdKey, returnvalue)
+  moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
+end

package/dist/node_modules/bullmq/dist/esm/commands/isFinished-3.lua

@@ -0,0 +1,48 @@
+--[[
+  Checks if a job is finished (.i.e. is in the completed or failed set)
+
+  Input: 
+    KEYS[1] completed key
+    KEYS[2] failed key
+    KEYS[3] job key
+
+    ARGV[1] job id
+    ARGV[2] return value?
+  Output:
+    0 - Not finished.
+    1 - Completed.
+    2 - Failed.
+   -1 - Missing job. 
+]]
+local rcall = redis.call
+if rcall("EXISTS", KEYS[3]) ~= 1 then
+  if ARGV[2] == "1" then
+
+    return {-1,"Missing key for job " .. KEYS[3] .. ". isFinished"}
+  end  
+  return -1
+end
+
+if rcall("ZSCORE", KEYS[1], ARGV[1]) ~= false then
+  if ARGV[2] == "1" then
+    local returnValue = rcall("HGET", KEYS[3], "returnvalue")
+
+    return {1,returnValue}
+  end
+  return 1
+end
+
+if rcall("ZSCORE", KEYS[2], ARGV[1]) ~= false then
+  if ARGV[2] == "1" then
+    local failedReason = rcall("HGET", KEYS[3], "failedReason")
+
+    return {2,failedReason}
+  end
+  return 2
+end
+
+if ARGV[2] == "1" then
+  return {0}
+end
+
+return 0

package/dist/node_modules/bullmq/dist/esm/commands/isJobInList-1.lua

@@ -0,0 +1,16 @@
+--[[
+  Checks if job is in a given list.
+
+  Input:
+    KEYS[1]
+    ARGV[1]
+
+  Output:
+    1 if element found in the list.
+]]
+
+-- Includes
+--- @include "includes/checkItemInList"
+
+local items = redis.call("LRANGE", KEYS[1] , 0, -1)
+return checkItemInList(items, ARGV[1])

package/dist/node_modules/bullmq/dist/esm/commands/isMaxed-2.lua

@@ -0,0 +1,17 @@
+--[[
+  Checks if queue is maxed.
+
+  Input:
+    KEYS[1] meta key
+    KEYS[2] active key
+
+  Output:
+    1 if element found in the list.
+]]
+
+local rcall = redis.call
+
+-- Includes
+--- @include "includes/isQueueMaxed"
+
+return isQueueMaxed(KEYS[1], KEYS[2])