@tachybase/plugin-adapter-bullmq 1.3.16 → 1.3.18

package/dist/node_modules/bullmq/dist/esm/scripts/removeUnprocessedChildren-2.js

@@ -0,0 +1,336 @@
+const content = `--[[
+    Remove a job from all the statuses it may be in as well as all its data.
+    In order to be able to remove a job, it cannot be active.
+    Input:
+      KEYS[1] jobKey
+      KEYS[2] meta key
+      ARGV[1] prefix
+      ARGV[2] jobId
+    Events:
+      'removed' for every children removed
+]]
+-- Includes
+--[[
+    Remove a job from all the statuses it may be in as well as all its data,
+    including its children. Active children can be ignored.
+    Events:
+      'removed'
+]]
+local rcall = redis.call
+-- Includes
+--[[
+  Functions to destructure job key.
+  Just a bit of warning, these functions may be a bit slow and affect performance significantly.
+]]
+local getJobIdFromKey = function (jobKey)
+  return string.match(jobKey, ".*:(.*)")
+end
+local getJobKeyPrefix = function (jobKey, jobId)
+  return string.sub(jobKey, 0, #jobKey - #jobId)
+end
+--[[
+  Function to get max events value or set by default 10000.
+]]
+local function getOrSetMaxEvents(metaKey)
+  local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
+  if not maxEvents then
+    maxEvents = 10000
+    rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
+  end
+  return maxEvents
+end
+--[[
+  Function to check if the job belongs to a job scheduler and
+  current delayed job matches with jobId
+]]
+local function isJobSchedulerJob(jobId, jobKey, jobSchedulersKey)
+  local repeatJobKey = rcall("HGET", jobKey, "rjk")
+  if repeatJobKey  then
+    local prevMillis = rcall("ZSCORE", jobSchedulersKey, repeatJobKey)
+    if prevMillis then
+      local currentDelayedJobId = "repeat:" .. repeatJobKey .. ":" .. prevMillis
+      return jobId == currentDelayedJobId
+    end
+  end
+  return false
+end
+--[[
+  Function to remove deduplication key if needed
+  when a job is being removed.
+]]
+local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
+  jobKey, jobId)
+  local deduplicationId = rcall("HGET", jobKey, "deid")
+  if deduplicationId then
+    local deduplicationKey = prefixKey .. "de:" .. deduplicationId
+    local currentJobId = rcall('GET', deduplicationKey)
+    if currentJobId and currentJobId == jobId then
+      return rcall("DEL", deduplicationKey)
+    end
+  end
+end
+--[[
+  Function to remove from any state.
+  returns:
+    prev state
+]]
+local function removeJobFromAnyState( prefix, jobId)
+  -- We start with the ZSCORE checks, since they have O(1) complexity
+  if rcall("ZSCORE", prefix .. "completed", jobId) then
+    rcall("ZREM", prefix .. "completed", jobId)
+    return "completed"
+  elseif rcall("ZSCORE", prefix .. "waiting-children", jobId) then
+    rcall("ZREM", prefix .. "waiting-children", jobId)
+    return "waiting-children"
+  elseif rcall("ZSCORE", prefix .. "delayed", jobId) then
+    rcall("ZREM", prefix .. "delayed", jobId)
+    return "delayed"
+  elseif rcall("ZSCORE", prefix .. "failed", jobId) then
+    rcall("ZREM", prefix .. "failed", jobId)
+    return "failed"
+  elseif rcall("ZSCORE", prefix .. "prioritized", jobId) then
+    rcall("ZREM", prefix .. "prioritized", jobId)
+    return "prioritized"
+  -- We remove only 1 element from the list, since we assume they are not added multiple times
+  elseif rcall("LREM", prefix .. "wait", 1, jobId) == 1 then
+    return "wait"
+  elseif rcall("LREM", prefix .. "paused", 1, jobId) == 1 then
+    return "paused"
+  elseif rcall("LREM", prefix .. "active", 1, jobId) == 1 then
+    return "active"
+  end
+  return "unknown"
+end
+--[[
+  Function to remove job keys.
+]]
+local function removeJobKeys(jobKey)
+  return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
+    jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
+end
+--[[
+  Check if this job has a parent. If so we will just remove it from
+  the parent child list, but if it is the last child we should move the parent to "wait/paused"
+  which requires code from "moveToFinished"
+]]
+-- Includes
+--[[
+  Function to add job in target list and add marker if needed.
+]]
+-- Includes
+--[[
+  Add marker if needed when a job is available.
+]]
+local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
+  if not isPausedOrMaxed then
+    rcall("ZADD", markerKey, 0, "0")
+  end  
+end
+local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
+  rcall(pushCmd, targetKey, jobId)
+  addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
+end
+--[[
+  Function to check for the meta.paused key to decide if we are paused or not
+  (since an empty list and !EXISTS are not really the same).
+]]
+local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
+  local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
+  if queueAttributes[1] then
+    return pausedKey, true
+  else
+    if queueAttributes[2] then
+      local activeCount = rcall("LLEN", activeKey)
+      if activeCount >= tonumber(queueAttributes[2]) then
+        return waitKey, true
+      else
+        return waitKey, false
+      end
+    end
+  end
+  return waitKey, false
+end
+local function _moveParentToWait(parentPrefix, parentId, emitEvent)
+  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
+    parentPrefix .. "wait", parentPrefix .. "paused")
+  addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
+  if emitEvent then
+    local parentEventStream = parentPrefix .. "events"
+    rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
+  end
+end
+local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debounceId)
+  if parentKey then
+    local parentDependenciesKey = parentKey .. ":dependencies"
+    local result = rcall("SREM", parentDependenciesKey, jobKey)
+    if result > 0 then
+      local pendingDependencies = rcall("SCARD", parentDependenciesKey)
+      if pendingDependencies == 0 then
+        local parentId = getJobIdFromKey(parentKey)
+        local parentPrefix = getJobKeyPrefix(parentKey, parentId)
+        local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
+        if numRemovedElements == 1 then
+          if hard then -- remove parent in same queue
+            if parentPrefix == baseKey then
+              removeParentDependencyKey(parentKey, hard, nil, baseKey, nil)
+              removeJobKeys(parentKey)
+              if debounceId then
+                rcall("DEL", parentPrefix .. "de:" .. debounceId)
+              end
+            else
+              _moveParentToWait(parentPrefix, parentId)
+            end
+          else
+            _moveParentToWait(parentPrefix, parentId, true)
+          end
+        end
+      end
+      return true
+    end
+  else
+    local parentAttributes = rcall("HMGET", jobKey, "parentKey", "deid")
+    local missedParentKey = parentAttributes[1]
+    if( (type(missedParentKey) == "string") and missedParentKey ~= ""
+      and (rcall("EXISTS", missedParentKey) == 1)) then
+      local parentDependenciesKey = missedParentKey .. ":dependencies"
+      local result = rcall("SREM", parentDependenciesKey, jobKey)
+      if result > 0 then
+        local pendingDependencies = rcall("SCARD", parentDependenciesKey)
+        if pendingDependencies == 0 then
+          local parentId = getJobIdFromKey(missedParentKey)
+          local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
+          local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
+          if numRemovedElements == 1 then
+            if hard then
+              if parentPrefix == baseKey then
+                removeParentDependencyKey(missedParentKey, hard, nil, baseKey, nil)
+                removeJobKeys(missedParentKey)
+                if parentAttributes[2] then
+                  rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
+                end
+              else
+                _moveParentToWait(parentPrefix, parentId)
+              end
+            else
+              _moveParentToWait(parentPrefix, parentId, true)
+            end
+          end
+        end
+        return true
+      end
+    end
+  end
+  return false
+end
+--[[
+  Function to recursively check if there are no locks
+  on the jobs to be removed.
+  returns:
+    boolean
+]]
+local function isLocked( prefix, jobId, removeChildren)
+  local jobKey = prefix .. jobId;
+  -- Check if this job is locked
+  local lockKey = jobKey .. ':lock'
+  local lock = rcall("GET", lockKey)
+  if not lock then
+    if removeChildren == "1" then
+      local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
+      if (#dependencies > 0) then
+        for i, childJobKey in ipairs(dependencies) do
+          -- We need to get the jobId for this job.
+          local childJobId = getJobIdFromKey(childJobKey)
+          local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
+          local result = isLocked( childJobPrefix, childJobId, removeChildren )
+          if result then
+            return true
+          end
+        end
+      end
+    end
+    return false
+  end
+  return true
+end
+local removeJobChildren
+local removeJobWithChildren
+removeJobChildren = function(prefix, jobKey, options)
+    -- Check if this job has children
+    -- If so, we are going to try to remove the children recursively in a depth-first way
+    -- because if some job is locked, we must exit with an error.
+    if not options.ignoreProcessed then
+        local processed = rcall("HGETALL", jobKey .. ":processed")
+        if #processed > 0 then
+            for i = 1, #processed, 2 do
+                local childJobId = getJobIdFromKey(processed[i])
+                local childJobPrefix = getJobKeyPrefix(processed[i], childJobId)
+                removeJobWithChildren(childJobPrefix, childJobId, jobKey, options)
+            end
+        end
+        local failed = rcall("HGETALL", jobKey .. ":failed")
+        if #failed > 0 then
+            for i = 1, #failed, 2 do
+                local childJobId = getJobIdFromKey(failed[i])
+                local childJobPrefix = getJobKeyPrefix(failed[i], childJobId)
+                removeJobWithChildren(childJobPrefix, childJobId, jobKey, options)
+            end
+        end
+        local unsuccessful = rcall("ZRANGE", jobKey .. ":unsuccessful", 0, -1)
+        if #unsuccessful > 0 then
+            for i = 1, #unsuccessful, 1 do
+                local childJobId = getJobIdFromKey(unsuccessful[i])
+                local childJobPrefix = getJobKeyPrefix(unsuccessful[i], childJobId)
+                removeJobWithChildren(childJobPrefix, childJobId, jobKey, options)
+            end
+        end
+    end
+    local dependencies = rcall("SMEMBERS", jobKey .. ":dependencies")
+    if #dependencies > 0 then
+        for i, childJobKey in ipairs(dependencies) do
+            local childJobId = getJobIdFromKey(childJobKey)
+            local childJobPrefix = getJobKeyPrefix(childJobKey, childJobId)
+            removeJobWithChildren(childJobPrefix, childJobId, jobKey, options)
+        end
+    end
+end
+removeJobWithChildren = function(prefix, jobId, parentKey, options)
+    local jobKey = prefix .. jobId
+    if options.ignoreLocked then
+        if isLocked(prefix, jobId) then
+            return
+        end
+    end
+    -- Check if job is in the failed zset
+    local failedSet = prefix .. "failed"
+    if not (options.ignoreProcessed and rcall("ZSCORE", failedSet, jobId)) then
+        removeParentDependencyKey(jobKey, false, parentKey, nil)
+        if options.removeChildren then
+            removeJobChildren(prefix, jobKey, options)
+        end
+        local prev = removeJobFromAnyState(prefix, jobId)
+        removeDeduplicationKeyIfNeededOnRemoval(prefix, jobKey, jobId)
+        if removeJobKeys(jobKey) > 0 then
+            local metaKey = prefix .. "meta"
+            local maxEvents = getOrSetMaxEvents(metaKey)
+            rcall("XADD", prefix .. "events", "MAXLEN", "~", maxEvents, "*", "event", "removed",
+                "jobId", jobId, "prev", prev)
+        end
+    end
+end
+local prefix = ARGV[1]
+local jobId = ARGV[2]
+local jobKey = KEYS[1]
+local metaKey = KEYS[2]
+local options = {
+  removeChildren = "1",
+  ignoreProcessed = true,
+  ignoreLocked = true
+}
+removeJobChildren(prefix, jobKey, options) 
+`;
+export const removeUnprocessedChildren = {
+    name: 'removeUnprocessedChildren',
+    content,
+    keys: 2,
+};
+//# sourceMappingURL=removeUnprocessedChildren-2.js.map

package/dist/node_modules/bullmq/dist/esm/scripts/reprocessJob-8.js

@@ -40,12 +40,12 @@ end
   Function to get max events value or set by default 10000.
 ]]
 local function getOrSetMaxEvents(metaKey)
-    local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
-    if not maxEvents then
-        maxEvents = 10000
-        rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
-    end
-    return maxEvents
+  local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
+  if not maxEvents then
+    maxEvents = 10000
+    rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
+  end
+  return maxEvents
 end
 --[[
   Function to check for the meta.paused key to decide if we are paused or not

package/dist/node_modules/bullmq/dist/esm/scripts/retryJob-11.js

@@ -17,6 +17,7 @@ const content = `--[[
       ARGV[3]  pushCmd
       ARGV[4]  jobId
       ARGV[5]  token
+      ARGV[6]  optional job fields to update
     Events:
       'waiting'
     Output:
@@ -47,10 +48,16 @@ end
   Function to add job considering priority.
 ]]
 -- Includes
+--[[
+  Function to get priority score.
+]]
+local function getPriorityScore(priority, priorityCounterKey)
+  local prioCounter = rcall("INCR", priorityCounterKey)
+  return priority * 0x100000000 + prioCounter % 0x100000000
+end
 local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
   isPausedOrMaxed)
-  local prioCounter = rcall("INCR", priorityCounterKey)
-  local score = priority * 0x100000000 + prioCounter % 0x100000000
+  local score = getPriorityScore(priority, priorityCounterKey)
   rcall("ZADD", prioritizedKey, score, jobId)
   addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
 end
@@ -58,12 +65,12 @@ end
   Function to get max events value or set by default 10000.
 ]]
 local function getOrSetMaxEvents(metaKey)
-    local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
-    if not maxEvents then
-        maxEvents = 10000
-        rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
-    end
-    return maxEvents
+  local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
+  if not maxEvents then
+    maxEvents = 10000
+    rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
+  end
+  return maxEvents
 end
 --[[
   Function to check for the meta.paused key to decide if we are paused or not
@@ -85,6 +92,22 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
   end
   return waitKey, false
 end
+--[[
+  Function to check if queue is paused or maxed
+  (since an empty list and !EXISTS are not really the same).
+]]
+local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
+  local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
+  if queueAttributes[1] then
+    return true
+  else
+    if queueAttributes[2] then
+      local activeCount = rcall("LLEN", activeKey)
+      return activeCount >= tonumber(queueAttributes[2])
+    end
+  end
+  return false
+end
 --[[
   Updates the delay set, by moving delayed jobs that should
   be processed now to "wait".
@@ -104,16 +127,17 @@ local function promoteDelayedJobs(delayedKey, markerKey, targetKey, prioritizedK
                 tonumber(rcall("HGET", jobKey, "priority")) or 0
             if priority == 0 then
                 -- LIFO or FIFO
-                addJobInTargetList(targetKey, markerKey, "LPUSH", isPaused, jobId)
+                rcall("LPUSH", targetKey, jobId)
             else
-                addJobWithPriority(markerKey, prioritizedKey, priority,
-                  jobId, priorityCounterKey, isPaused)
+                local score = getPriorityScore(priority, priorityCounterKey)
+                rcall("ZADD", prioritizedKey, score, jobId)
             end
             -- Emit waiting event
             rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
                   jobId, "prev", "delayed")
             rcall("HSET", jobKey, "delay", 0)
         end
+        addBaseMarkerIfNeeded(markerKey, isPaused)
     end
 end
 local function removeLock(jobKey, stalledKey, token, jobId)
@@ -136,34 +160,31 @@ local function removeLock(jobKey, stalledKey, token, jobId)
   return 0
 end
 --[[
-  Function to check if queue is paused or maxed
-  (since an empty list and !EXISTS are not really the same).
+  Function to update a bunch of fields in a job.
 ]]
-local function isQueuePausedOrMaxed(queueMetaKey, activeKey)
-  local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
-  if queueAttributes[1] then
-    return true
-  else
-    if queueAttributes[2] then
-      local activeCount = rcall("LLEN", activeKey)
-      return activeCount >= tonumber(queueAttributes[2])
+local function updateJobFields(jobKey, msgpackedFields)
+  if msgpackedFields and #msgpackedFields > 0 then
+    local fieldsToUpdate = cmsgpack.unpack(msgpackedFields)
+    if fieldsToUpdate then
+      rcall("HMSET", jobKey, unpack(fieldsToUpdate))
     end
   end
-  return false
 end
 local target, isPausedOrMaxed = getTargetQueueList(KEYS[5], KEYS[1], KEYS[2], KEYS[3])
 local markerKey = KEYS[10]
 -- Check if there are delayed jobs that we can move to wait.
 -- test example: when there are delayed jobs between retries
 promoteDelayedJobs(KEYS[7], markerKey, target, KEYS[8], KEYS[6], ARGV[1], ARGV[2], KEYS[9], isPausedOrMaxed)
-if rcall("EXISTS", KEYS[4]) == 1 then
-  local errorCode = removeLock(KEYS[4], KEYS[11], ARGV[5], ARGV[4]) 
+local jobKey = KEYS[4]
+if rcall("EXISTS", jobKey) == 1 then
+  local errorCode = removeLock(jobKey, KEYS[11], ARGV[5], ARGV[4]) 
   if errorCode < 0 then
     return errorCode
   end
+  updateJobFields(jobKey, ARGV[6])
   local numRemovedElements = rcall("LREM", KEYS[1], -1, ARGV[4])
   if (numRemovedElements < 1) then return -3 end
-  local priority = tonumber(rcall("HGET", KEYS[4], "priority")) or 0
+  local priority = tonumber(rcall("HGET", jobKey, "priority")) or 0
   --need to re-evaluate after removing job from active
   isPausedOrMaxed = isQueuePausedOrMaxed(KEYS[5], KEYS[1])
   -- Standard or priority add
@@ -172,7 +193,7 @@ if rcall("EXISTS", KEYS[4]) == 1 then
   else
     addJobWithPriority(markerKey, KEYS[8], priority, ARGV[4], KEYS[9], isPausedOrMaxed)
   end
-  rcall("HINCRBY", KEYS[4], "atm", 1)
+  rcall("HINCRBY", jobKey, "atm", 1)
   local maxEvents = getOrSetMaxEvents(KEYS[5])
   -- Emit waiting event
   rcall("XADD", KEYS[6], "MAXLEN", "~", maxEvents, "*", "event", "waiting",

package/dist/node_modules/bullmq/dist/esm/scripts/updateJobScheduler-12.d.ts

@@ -0,0 +1,5 @@
+export declare const updateJobScheduler: {
+    name: string;
+    content: string;
+    keys: number;
+};