@tachybase/plugin-adapter-bullmq 1.3.17 → 1.3.19

package/dist/node_modules/bullmq/dist/cjs/interfaces/index.js

@@ -6,7 +6,6 @@ tslib_1.__exportStar(require("./backoff-options"), exports);
 tslib_1.__exportStar(require("./base-job-options"), exports);
 tslib_1.__exportStar(require("./child-message"), exports);
 tslib_1.__exportStar(require("./connection"), exports);
-tslib_1.__exportStar(require("./debounce-options"), exports);
 tslib_1.__exportStar(require("./flow-job"), exports);
 tslib_1.__exportStar(require("./ioredis-events"), exports);
 tslib_1.__exportStar(require("./job-json"), exports);
@@ -17,6 +16,7 @@ tslib_1.__exportStar(require("./metrics"), exports);
 tslib_1.__exportStar(require("./minimal-job"), exports);
 tslib_1.__exportStar(require("./parent-message"), exports);
 tslib_1.__exportStar(require("./parent"), exports);
+tslib_1.__exportStar(require("./parent-options"), exports);
 tslib_1.__exportStar(require("./queue-options"), exports);
 tslib_1.__exportStar(require("./rate-limiter-options"), exports);
 tslib_1.__exportStar(require("./redis-options"), exports);
@@ -29,4 +29,5 @@ tslib_1.__exportStar(require("./sandboxed-job"), exports);
 tslib_1.__exportStar(require("./sandboxed-options"), exports);
 tslib_1.__exportStar(require("./worker-options"), exports);
 tslib_1.__exportStar(require("./telemetry"), exports);
+tslib_1.__exportStar(require("./receiver"), exports);
 //# sourceMappingURL=index.js.map

package/dist/node_modules/bullmq/dist/cjs/interfaces/{debounce-options.js → parent-options.js}

@@ -1,3 +1,3 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
-//# sourceMappingURL=debounce-options.js.map
+//# sourceMappingURL=parent-options.js.map

package/dist/node_modules/bullmq/dist/cjs/interfaces/receiver.js

@@ -0,0 +1,3 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+//# sourceMappingURL=receiver.js.map

package/dist/node_modules/bullmq/dist/cjs/scripts/addDelayedJob-6.js

@@ -48,6 +48,13 @@ local repeatJobKey = args[9]
 local deduplicationKey = args[10]
 local parentData
 -- Includes
+--[[
+  Adds a delayed job to the queue by doing the following:
+    - Creates a new job key with the job data.
+    - adds to delayed zset.
+    - Emits a global event 'delayed' if the job is delayed.
+]]
+-- Includes
 --[[
   Add delay marker if needed.
 ]]
@@ -59,7 +66,7 @@ local function getNextDelayedTimestamp(delayedKey)
   local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
   if #result then
     local nextTimestamp = tonumber(result[2])
-    if nextTimestamp ~= nil then 
+    if nextTimestamp ~= nil then
       return nextTimestamp / 0x1000
     end
   end
@@ -72,29 +79,6 @@ local function addDelayMarkerIfNeeded(markerKey, delayedKey)
     rcall("ZADD", markerKey, nextTimestamp, "1")
   end
 end
---[[
-  Function to debounce a job.
-]]
-local function deduplicateJob(prefixKey, deduplicationOpts, jobId, deduplicationKey, eventsKey, maxEvents)
-  local deduplicationId = deduplicationOpts and deduplicationOpts['id']
-  if deduplicationId then
-    local ttl = deduplicationOpts['ttl']
-    local deduplicationKeyExists
-    if ttl then
-      deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'PX', ttl, 'NX')
-    else
-      deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'NX')
-    end
-    if deduplicationKeyExists then
-      local currentDebounceJobId = rcall('GET', deduplicationKey)
-      rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
-        "debounced", "jobId", currentDebounceJobId, "debounceId", deduplicationId)
-      rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
-        "deduplicated", "jobId", currentDebounceJobId, "deduplicationId", deduplicationId)
-      return currentDebounceJobId
-    end
-  end
-end
 --[[
   Bake in the job id first 12 bits into the timestamp
   to guarantee correct execution order of delayed jobs
@@ -119,16 +103,97 @@ local function getDelayedScore(delayedKey, timestamp, delay)
   end
   return minScore, delayedTimestamp
 end
+local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
+  maxEvents, markerKey, delay)
+  local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
+  rcall("ZADD", delayedKey, score, jobId)
+  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
+    "jobId", jobId, "delay", delayedTimestamp)
+  -- mark that a delayed job is available
+  addDelayMarkerIfNeeded(markerKey, delayedKey)
+end
+--[[
+  Function to debounce a job.
+]]
+-- Includes
+--[[
+  Function to remove job keys.
+]]
+local function removeJobKeys(jobKey)
+  return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
+    jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
+end
+local function deduplicateJob(deduplicationOpts, jobId, delayedKey, deduplicationKey, eventsKey, maxEvents,
+    prefix)
+    local deduplicationId = deduplicationOpts and deduplicationOpts['id']
+    if deduplicationId then
+        local ttl = deduplicationOpts['ttl']
+        if deduplicationOpts['replace'] and ttl and ttl > 0 then
+            local currentDebounceJobId = rcall('GET', deduplicationKey)
+            if currentDebounceJobId then
+                if rcall("ZREM", delayedKey, currentDebounceJobId) > 0 then
+                    removeJobKeys(prefix .. currentDebounceJobId)
+                    rcall("XADD", eventsKey, "*", "event", "removed", "jobId", currentDebounceJobId,
+                        "prev", "delayed")
+                    if deduplicationOpts['extend'] then
+                        rcall('SET', deduplicationKey, jobId, 'PX', ttl)
+                    else
+                        rcall('SET', deduplicationKey, jobId, 'KEEPTTL')
+                    end
+                    rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
+                        jobId, "deduplicationId", deduplicationId, "deduplicatedJobId", currentDebounceJobId)
+                    return
+                else
+                    return currentDebounceJobId
+                end
+            else
+                rcall('SET', deduplicationKey, jobId, 'PX', ttl)
+                return
+            end
+        else
+            local ttl = deduplicationOpts['ttl']
+            local deduplicationKeyExists
+            if ttl then
+                if deduplicationOpts['extend'] then
+                    local currentDebounceJobId = rcall('GET', deduplicationKey)
+                    if currentDebounceJobId then
+                        rcall('SET', deduplicationKey, currentDebounceJobId, 'PX', ttl)
+                        rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced",
+                            "jobId", currentDebounceJobId, "debounceId", deduplicationId)
+                        rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
+                            currentDebounceJobId, "deduplicationId", deduplicationId, "deduplicatedJobId", jobId)
+                        return currentDebounceJobId
+                    else
+                        rcall('SET', deduplicationKey, jobId, 'PX', ttl)
+                        return
+                    end
+                else
+                    deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'PX', ttl, 'NX')
+                end
+            else
+                deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'NX')
+            end
+            if deduplicationKeyExists then
+                local currentDebounceJobId = rcall('GET', deduplicationKey)
+                rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "debounced", "jobId",
+                    currentDebounceJobId, "debounceId", deduplicationId)
+                rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "deduplicated", "jobId",
+                    currentDebounceJobId, "deduplicationId", deduplicationId, "deduplicatedJobId", jobId)
+                return currentDebounceJobId
+            end
+        end
+    end
+end
 --[[
   Function to get max events value or set by default 10000.
 ]]
 local function getOrSetMaxEvents(metaKey)
-    local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
-    if not maxEvents then
-        maxEvents = 10000
-        rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
-    end
-    return maxEvents
+  local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
+  if not maxEvents then
+    maxEvents = 10000
+    rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
+  end
+  return maxEvents
 end
 --[[
   Function to handle the case when job is duplicated.
@@ -146,7 +211,16 @@ end
 ]]
 -- Includes
 --[[
-  Validate and move parent to active if needed.
+  Validate and move parent to a wait status (waiting, delayed or prioritized)
+  if no pending dependencies.
+]]
+-- Includes
+--[[
+  Validate and move parent to a wait status (waiting, delayed or prioritized) if needed.
+]]
+-- Includes
+--[[
+  Move parent to a wait status (wait, prioritized or delayed)
 ]]
 -- Includes
 --[[
@@ -169,10 +243,16 @@ end
   Function to add job considering priority.
 ]]
 -- Includes
+--[[
+  Function to get priority score.
+]]
+local function getPriorityScore(priority, priorityCounterKey)
+  local prioCounter = rcall("INCR", priorityCounterKey)
+  return priority * 0x100000000 + prioCounter % 0x100000000
+end
 local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
   isPausedOrMaxed)
-  local prioCounter = rcall("INCR", priorityCounterKey)
-  local score = priority * 0x100000000 + prioCounter % 0x100000000
+  local score = getPriorityScore(priority, priorityCounterKey)
   rcall("ZADD", prioritizedKey, score, jobId)
   addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
 end
@@ -212,57 +292,64 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
   end
   return waitKey, false
 end
-local function moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey,
-                                        parentKey, parentId, timestamp)
-    local isParentActive = rcall("ZSCORE",
-                                 parentQueueKey .. ":waiting-children", parentId)
-    if rcall("SCARD", parentDependenciesKey) == 0 and isParentActive then
-        rcall("ZREM", parentQueueKey .. ":waiting-children", parentId)
-        local parentWaitKey = parentQueueKey .. ":wait"
-        local parentPausedKey = parentQueueKey .. ":paused"
-        local parentActiveKey = parentQueueKey .. ":active"
-        local parentMetaKey = parentQueueKey .. ":meta"
-        local parentMarkerKey = parentQueueKey .. ":marker"
-        local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
-        local priority = tonumber(jobAttributes[1]) or 0
-        local delay = tonumber(jobAttributes[2]) or 0
-        if delay > 0 then
-            local delayedTimestamp = tonumber(timestamp) + delay
-            local score = delayedTimestamp * 0x1000
-            local parentDelayedKey = parentQueueKey .. ":delayed"
-            rcall("ZADD", parentDelayedKey, score, parentId)
-            rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed",
-                  "jobId", parentId, "delay", delayedTimestamp)
-            addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
+local function moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
+    local parentWaitKey = parentQueueKey .. ":wait"
+    local parentPausedKey = parentQueueKey .. ":paused"
+    local parentActiveKey = parentQueueKey .. ":active"
+    local parentMetaKey = parentQueueKey .. ":meta"
+    local parentMarkerKey = parentQueueKey .. ":marker"
+    local jobAttributes = rcall("HMGET", parentKey, "priority", "delay")
+    local priority = tonumber(jobAttributes[1]) or 0
+    local delay = tonumber(jobAttributes[2]) or 0
+    if delay > 0 then
+        local delayedTimestamp = tonumber(timestamp) + delay
+        local score = delayedTimestamp * 0x1000
+        local parentDelayedKey = parentQueueKey .. ":delayed"
+        rcall("ZADD", parentDelayedKey, score, parentId)
+        rcall("XADD", parentQueueKey .. ":events", "*", "event", "delayed", "jobId", parentId, "delay",
+            delayedTimestamp)
+        addDelayMarkerIfNeeded(parentMarkerKey, parentDelayedKey)
+    else
+        if priority == 0 then
+            local parentTarget, isParentPausedOrMaxed = getTargetQueueList(parentMetaKey, parentActiveKey,
+                parentWaitKey, parentPausedKey)
+            addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed, parentId)
         else
-            if priority == 0 then
-                local parentTarget, isParentPausedOrMaxed =
-                    getTargetQueueList(parentMetaKey, parentActiveKey, parentWaitKey,
-                                       parentPausedKey)
-                addJobInTargetList(parentTarget, parentMarkerKey, "RPUSH", isParentPausedOrMaxed,
-                    parentId)
-            else
-                local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
-                addJobWithPriority(parentMarkerKey,
-                                   parentQueueKey .. ":prioritized", priority,
-                                   parentId, parentQueueKey .. ":pc", isPausedOrMaxed)
-            end
-            rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting",
-                  "jobId", parentId, "prev", "waiting-children")
+            local isPausedOrMaxed = isQueuePausedOrMaxed(parentMetaKey, parentActiveKey)
+            addJobWithPriority(parentMarkerKey, parentQueueKey .. ":prioritized", priority, parentId,
+                parentQueueKey .. ":pc", isPausedOrMaxed)
         end
+        rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev",
+            "waiting-children")
+    end
+end
+local function moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
+  if rcall("EXISTS", parentKey) == 1 then
+    local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
+    if rcall("ZSCORE", parentWaitingChildrenKey, parentId) then    
+      rcall("ZREM", parentWaitingChildrenKey, parentId)
+      moveParentToWait(parentQueueKey, parentKey, parentId, timestamp)
     end
+  end
+end
+local function moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey,
+  parentId, timestamp)
+  local doNotHavePendingDependencies = rcall("SCARD", parentDependenciesKey) == 0
+  if doNotHavePendingDependencies then
+    moveParentToWaitIfNeeded(parentQueueKey, parentKey, parentId, timestamp)
+  end
 end
 local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDependenciesKey,
   parentId, jobIdKey, returnvalue, timestamp )
   local processedSet = parentKey .. ":processed"
   rcall("HSET", processedSet, jobIdKey, returnvalue)
-  moveParentToWaitIfNeeded(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
+  moveParentToWaitIfNoPendingDependencies(parentQueueKey, parentDependenciesKey, parentKey, parentId, timestamp)
 end
 local function updateExistingJobsParent(parentKey, parent, parentData,
                                         parentDependenciesKey, completedKey,
                                         jobIdKey, jobId, timestamp)
     if parentKey ~= nil then
-        if rcall("ZSCORE", completedKey, jobId) ~= false then
+        if rcall("ZSCORE", completedKey, jobId) then
             local returnvalue = rcall("HGET", jobIdKey, "returnvalue")
             updateParentDepsIfNeeded(parentKey, parent['queueKey'],
                                      parentDependenciesKey, parent['id'],
@@ -308,7 +395,7 @@ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
         table.insert(optionalValues, "parent")
         table.insert(optionalValues, parentData)
     end
-    if repeatJobKey ~= nil then
+    if repeatJobKey then
         table.insert(optionalValues, "rjk")
         table.insert(optionalValues, repeatJobKey)
     end
@@ -343,22 +430,14 @@ else
             maxEvents, timestamp)
     end
 end
-local deduplicationJobId = deduplicateJob(args[1], opts['de'],
-  jobId, deduplicationKey, eventsKey, maxEvents)
+local deduplicationJobId = deduplicateJob(opts['de'], jobId, delayedKey, deduplicationKey,
+  eventsKey, maxEvents, args[1])
 if deduplicationJobId then
   return deduplicationJobId
 end
--- Store the job.
 local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
-                                 opts, timestamp, parentKey, parentData,
-                                 repeatJobKey)
-local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
-rcall("ZADD", delayedKey, score, jobId)
-rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
-      "jobId", jobId, "delay", delayedTimestamp)
--- mark that a delayed job is available
-local markerKey = KEYS[1]
-addDelayMarkerIfNeeded(markerKey, delayedKey)
+    opts, timestamp, parentKey, parentData, repeatJobKey)
+addDelayedJob(jobId, delayedKey, eventsKey, timestamp, maxEvents, KEYS[1], delay)
 -- Check if this job is a child of another job, if so add it to the parents dependencies
 if parentDependenciesKey ~= nil then
     rcall("SADD", parentDependenciesKey, jobIdKey)