@tachybase/plugin-adapter-bullmq 1.3.16 → 1.3.18

package/dist/node_modules/bullmq/dist/esm/scripts/addJobScheduler-11.js

@@ -0,0 +1,462 @@
+const content = `--[[
+  Adds a job scheduler, i.e. a job factory that creates jobs based on a given schedule (repeat options).
+    Input:
+      KEYS[1]  'repeat' key
+      KEYS[2]  'delayed' key
+      KEYS[3]  'wait' key
+      KEYS[4]  'paused' key
+      KEYS[5]  'meta' key
+      KEYS[6]  'prioritized' key
+      KEYS[7]  'marker' key
+      KEYS[8]  'id' key
+      KEYS[9]  'events' key
+      KEYS[10] 'pc' priority counter
+      KEYS[11] 'active' key
+      ARGV[1] next milliseconds
+      ARGV[2] msgpacked options
+            [1]  name
+            [2]  tz?
+            [3]  patten?
+            [4]  endDate?
+            [5]  every?
+      ARGV[3] jobs scheduler id
+      ARGV[4] Json stringified template data
+      ARGV[5] mspacked template opts
+      ARGV[6] msgpacked delayed opts
+      ARGV[7] timestamp
+      ARGV[8] prefix key
+      ARGV[9] producer key
+      Output:
+        repeatableKey  - OK
+]]
+local rcall = redis.call
+local repeatKey = KEYS[1]
+local delayedKey = KEYS[2]
+local waitKey = KEYS[3]
+local pausedKey = KEYS[4]
+local metaKey = KEYS[5]
+local prioritizedKey = KEYS[6]
+local eventsKey = KEYS[9]
+local nextMillis = ARGV[1]
+local jobSchedulerId = ARGV[3]
+local templateOpts = cmsgpack.unpack(ARGV[5])
+local prefixKey = ARGV[8]
+-- Includes
+--[[
+  Add delay marker if needed.
+]]
+-- Includes
+--[[
+  Adds a delayed job to the queue by doing the following:
+    - Creates a new job key with the job data.
+    - adds to delayed zset.
+    - Emits a global event 'delayed' if the job is delayed.
+]]
+-- Includes
+--[[
+  Add delay marker if needed.
+]]
+-- Includes
+--[[
+  Function to return the next delayed job timestamp.
+]]
+local function getNextDelayedTimestamp(delayedKey)
+  local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
+  if #result then
+    local nextTimestamp = tonumber(result[2])
+    if nextTimestamp ~= nil then
+      return nextTimestamp / 0x1000
+    end
+  end
+end
+local function addDelayMarkerIfNeeded(markerKey, delayedKey)
+  local nextTimestamp = getNextDelayedTimestamp(delayedKey)
+  if nextTimestamp ~= nil then
+    -- Replace the score of the marker with the newest known
+    -- next timestamp.
+    rcall("ZADD", markerKey, nextTimestamp, "1")
+  end
+end
+--[[
+  Bake in the job id first 12 bits into the timestamp
+  to guarantee correct execution order of delayed jobs
+  (up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
+  WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
+]]
+local function getDelayedScore(delayedKey, timestamp, delay)
+  local delayedTimestamp = (delay > 0 and (tonumber(timestamp) + delay)) or tonumber(timestamp)
+  local minScore = delayedTimestamp * 0x1000
+  local maxScore = (delayedTimestamp + 1 ) * 0x1000 - 1
+  local result = rcall("ZREVRANGEBYSCORE", delayedKey, maxScore,
+    minScore, "WITHSCORES","LIMIT", 0, 1)
+  if #result then
+    local currentMaxScore = tonumber(result[2])
+    if currentMaxScore ~= nil then
+      if currentMaxScore >= maxScore then
+        return maxScore, delayedTimestamp
+      else
+        return currentMaxScore + 1, delayedTimestamp
+      end
+    end
+  end
+  return minScore, delayedTimestamp
+end
+local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
+  maxEvents, markerKey, delay)
+  local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
+  rcall("ZADD", delayedKey, score, jobId)
+  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
+    "jobId", jobId, "delay", delayedTimestamp)
+  -- mark that a delayed job is available
+  addDelayMarkerIfNeeded(markerKey, delayedKey)
+end
+--[[
+  Function to add job considering priority.
+]]
+-- Includes
+--[[
+  Add marker if needed when a job is available.
+]]
+local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
+  if not isPausedOrMaxed then
+    rcall("ZADD", markerKey, 0, "0")
+  end  
+end
+--[[
+  Function to get priority score.
+]]
+local function getPriorityScore(priority, priorityCounterKey)
+  local prioCounter = rcall("INCR", priorityCounterKey)
+  return priority * 0x100000000 + prioCounter % 0x100000000
+end
+local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
+  isPausedOrMaxed)
+  local score = getPriorityScore(priority, priorityCounterKey)
+  rcall("ZADD", prioritizedKey, score, jobId)
+  addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
+end
+--[[
+  Function to check for the meta.paused key to decide if we are paused or not
+  (since an empty list and !EXISTS are not really the same).
+]]
+local function isQueuePaused(queueMetaKey)
+  return rcall("HEXISTS", queueMetaKey, "paused") == 1
+end
+--[[
+  Function to store a job
+]]
+local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
+                        parentKey, parentData, repeatJobKey)
+    local jsonOpts = cjson.encode(opts)
+    local delay = opts['delay'] or 0
+    local priority = opts['priority'] or 0
+    local debounceId = opts['de'] and opts['de']['id']
+    local optionalValues = {}
+    if parentKey ~= nil then
+        table.insert(optionalValues, "parentKey")
+        table.insert(optionalValues, parentKey)
+        table.insert(optionalValues, "parent")
+        table.insert(optionalValues, parentData)
+    end
+    if repeatJobKey then
+        table.insert(optionalValues, "rjk")
+        table.insert(optionalValues, repeatJobKey)
+    end
+    if debounceId then
+        table.insert(optionalValues, "deid")
+        table.insert(optionalValues, debounceId)
+    end
+    rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
+          "timestamp", timestamp, "delay", delay, "priority", priority,
+          unpack(optionalValues))
+    rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
+    return delay, priority
+end
+--[[
+  Function to check for the meta.paused key to decide if we are paused or not
+  (since an empty list and !EXISTS are not really the same).
+]]
+local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
+  local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
+  if queueAttributes[1] then
+    return pausedKey, true
+  else
+    if queueAttributes[2] then
+      local activeCount = rcall("LLEN", activeKey)
+      if activeCount >= tonumber(queueAttributes[2]) then
+        return waitKey, true
+      else
+        return waitKey, false
+      end
+    end
+  end
+  return waitKey, false
+end
+--[[
+  Function to add job in target list and add marker if needed.
+]]
+-- Includes
+local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
+  rcall(pushCmd, targetKey, jobId)
+  addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
+end
+local function addJobFromScheduler(jobKey, jobId, rawOpts, waitKey, pausedKey, activeKey, metaKey, 
+  prioritizedKey, priorityCounter, delayedKey, markerKey, eventsKey, name, maxEvents, timestamp,
+  data, jobSchedulerId)
+  local opts = cmsgpack.unpack(rawOpts)
+  local delay, priority = storeJob(eventsKey, jobKey, jobId, name, data,
+    opts, timestamp, nil, nil, jobSchedulerId)
+  if delay ~= 0 then
+    addDelayedJob(jobId, delayedKey, eventsKey, timestamp, maxEvents, markerKey, delay)
+  else
+    local target, isPausedOrMaxed = getTargetQueueList(metaKey, activeKey, waitKey, pausedKey)
+    -- Standard or priority add
+    if priority == 0 then
+      local pushCmd = opts['lifo'] and 'RPUSH' or 'LPUSH'
+      addJobInTargetList(target, markerKey, pushCmd, isPausedOrMaxed, jobId)
+    else
+      -- Priority add
+      addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounter, isPausedOrMaxed)
+    end
+    -- Emit waiting event
+    rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents,  "*", "event", "waiting", "jobId", jobId)
+  end
+end
+--[[
+  Function to get max events value or set by default 10000.
+]]
+local function getOrSetMaxEvents(metaKey)
+  local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
+  if not maxEvents then
+    maxEvents = 10000
+    rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
+  end
+  return maxEvents
+end
+--[[
+  Function to remove job.
+]]
+-- Includes
+--[[
+  Function to remove deduplication key if needed
+  when a job is being removed.
+]]
+local function removeDeduplicationKeyIfNeededOnRemoval(prefixKey,
+  jobKey, jobId)
+  local deduplicationId = rcall("HGET", jobKey, "deid")
+  if deduplicationId then
+    local deduplicationKey = prefixKey .. "de:" .. deduplicationId
+    local currentJobId = rcall('GET', deduplicationKey)
+    if currentJobId and currentJobId == jobId then
+      return rcall("DEL", deduplicationKey)
+    end
+  end
+end
+--[[
+  Function to remove job keys.
+]]
+local function removeJobKeys(jobKey)
+  return rcall("DEL", jobKey, jobKey .. ':logs', jobKey .. ':dependencies',
+    jobKey .. ':processed', jobKey .. ':failed', jobKey .. ':unsuccessful')
+end
+--[[
+  Check if this job has a parent. If so we will just remove it from
+  the parent child list, but if it is the last child we should move the parent to "wait/paused"
+  which requires code from "moveToFinished"
+]]
+-- Includes
+--[[
+  Functions to destructure job key.
+  Just a bit of warning, these functions may be a bit slow and affect performance significantly.
+]]
+local getJobIdFromKey = function (jobKey)
+  return string.match(jobKey, ".*:(.*)")
+end
+local getJobKeyPrefix = function (jobKey, jobId)
+  return string.sub(jobKey, 0, #jobKey - #jobId)
+end
+local function _moveParentToWait(parentPrefix, parentId, emitEvent)
+  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
+    parentPrefix .. "wait", parentPrefix .. "paused")
+  addJobInTargetList(parentTarget, parentPrefix .. "marker", "RPUSH", isPausedOrMaxed, parentId)
+  if emitEvent then
+    local parentEventStream = parentPrefix .. "events"
+    rcall("XADD", parentEventStream, "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
+  end
+end
+local function removeParentDependencyKey(jobKey, hard, parentKey, baseKey, debounceId)
+  if parentKey then
+    local parentDependenciesKey = parentKey .. ":dependencies"
+    local result = rcall("SREM", parentDependenciesKey, jobKey)
+    if result > 0 then
+      local pendingDependencies = rcall("SCARD", parentDependenciesKey)
+      if pendingDependencies == 0 then
+        local parentId = getJobIdFromKey(parentKey)
+        local parentPrefix = getJobKeyPrefix(parentKey, parentId)
+        local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
+        if numRemovedElements == 1 then
+          if hard then -- remove parent in same queue
+            if parentPrefix == baseKey then
+              removeParentDependencyKey(parentKey, hard, nil, baseKey, nil)
+              removeJobKeys(parentKey)
+              if debounceId then
+                rcall("DEL", parentPrefix .. "de:" .. debounceId)
+              end
+            else
+              _moveParentToWait(parentPrefix, parentId)
+            end
+          else
+            _moveParentToWait(parentPrefix, parentId, true)
+          end
+        end
+      end
+      return true
+    end
+  else
+    local parentAttributes = rcall("HMGET", jobKey, "parentKey", "deid")
+    local missedParentKey = parentAttributes[1]
+    if( (type(missedParentKey) == "string") and missedParentKey ~= ""
+      and (rcall("EXISTS", missedParentKey) == 1)) then
+      local parentDependenciesKey = missedParentKey .. ":dependencies"
+      local result = rcall("SREM", parentDependenciesKey, jobKey)
+      if result > 0 then
+        local pendingDependencies = rcall("SCARD", parentDependenciesKey)
+        if pendingDependencies == 0 then
+          local parentId = getJobIdFromKey(missedParentKey)
+          local parentPrefix = getJobKeyPrefix(missedParentKey, parentId)
+          local numRemovedElements = rcall("ZREM", parentPrefix .. "waiting-children", parentId)
+          if numRemovedElements == 1 then
+            if hard then
+              if parentPrefix == baseKey then
+                removeParentDependencyKey(missedParentKey, hard, nil, baseKey, nil)
+                removeJobKeys(missedParentKey)
+                if parentAttributes[2] then
+                  rcall("DEL", parentPrefix .. "de:" .. parentAttributes[2])
+                end
+              else
+                _moveParentToWait(parentPrefix, parentId)
+              end
+            else
+              _moveParentToWait(parentPrefix, parentId, true)
+            end
+          end
+        end
+        return true
+      end
+    end
+  end
+  return false
+end
+local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
+  local jobKey = baseKey .. jobId
+  removeParentDependencyKey(jobKey, hard, nil, baseKey)
+  if shouldRemoveDeduplicationKey then
+    removeDeduplicationKeyIfNeededOnRemoval(baseKey, jobKey, jobId)
+  end
+  removeJobKeys(jobKey)
+end
+--[[
+  Function to store a job scheduler
+]]
+local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMillis, opts,
+  templateData, templateOpts)
+  rcall("ZADD", repeatKey, nextMillis, schedulerId)
+  local optionalValues = {}
+  if opts['tz'] then
+    table.insert(optionalValues, "tz")
+    table.insert(optionalValues, opts['tz'])
+  end
+  if opts['limit'] then
+    table.insert(optionalValues, "limit")
+    table.insert(optionalValues, opts['limit'])
+  end
+  if opts['pattern'] then
+    table.insert(optionalValues, "pattern")
+    table.insert(optionalValues, opts['pattern'])
+  end
+  if opts['endDate'] then
+    table.insert(optionalValues, "endDate")
+    table.insert(optionalValues, opts['endDate'])
+  end
+  if opts['every'] then
+    table.insert(optionalValues, "every")
+    table.insert(optionalValues, opts['every'])
+  end
+  if opts['offset'] then
+    table.insert(optionalValues, "offset")
+    table.insert(optionalValues, opts['offset'])
+  end
+  local jsonTemplateOpts = cjson.encode(templateOpts)
+  if jsonTemplateOpts and jsonTemplateOpts ~= '{}' then
+    table.insert(optionalValues, "opts")
+    table.insert(optionalValues, jsonTemplateOpts)
+  end
+  if templateData and templateData ~= '{}' then
+    table.insert(optionalValues, "data")
+    table.insert(optionalValues, templateData)
+  end
+  rcall("DEL", schedulerKey) -- remove all attributes and then re-insert new ones
+  rcall("HMSET", schedulerKey, "name", opts['name'], "ic", 1, unpack(optionalValues))
+end
+-- If we are overriding a repeatable job we must delete the delayed job for
+-- the next iteration.
+local schedulerKey = repeatKey .. ":" .. jobSchedulerId
+local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
+local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
+local maxEvents = getOrSetMaxEvents(metaKey)
+local function removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey, jobId,
+    metaKey, eventsKey)
+    if rcall("ZSCORE", delayedKey, jobId) then
+        removeJob(jobId, true, prefixKey, true --[[remove debounce key]] )
+        rcall("ZREM", delayedKey, jobId)
+        return true
+    elseif rcall("ZSCORE", prioritizedKey, jobId) then
+        removeJob(jobId, true, prefixKey, true --[[remove debounce key]] )
+        rcall("ZREM", prioritizedKey, jobId)
+        return true
+    else
+        local pausedOrWaitKey = waitKey
+        if isQueuePaused(metaKey) then
+            pausedOrWaitKey = pausedKey
+        end
+        if rcall("LREM", pausedOrWaitKey, 1, jobId) > 0 then
+            removeJob(jobId, true, prefixKey, true --[[remove debounce key]] )
+            return true
+        end
+    end
+    return false
+end
+if rcall("EXISTS", nextDelayedJobKey) == 1 then
+    if not removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey,
+        nextDelayedJobId, metaKey, eventsKey) then
+        rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
+            "duplicated", "jobId", nextDelayedJobId)
+        return nextDelayedJobId .. "" -- convert to string
+    end
+end
+local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
+if prevMillis then    
+    local currentJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
+    local currentDelayedJobKey = schedulerKey .. ":" .. prevMillis
+    if currentJobId ~= nextDelayedJobId and rcall("EXISTS", currentDelayedJobKey) == 1 then
+        removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey,
+            currentJobId, metaKey, eventsKey)
+    end
+end
+local schedulerOpts = cmsgpack.unpack(ARGV[2])
+storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, ARGV[4], templateOpts)
+rcall("INCR", KEYS[8])
+addJobFromScheduler(nextDelayedJobKey, nextDelayedJobId, ARGV[6], waitKey, pausedKey,
+    KEYS[11], metaKey, prioritizedKey, KEYS[10], delayedKey, KEYS[7], eventsKey,
+    schedulerOpts['name'], maxEvents, ARGV[7], ARGV[4], jobSchedulerId)
+if ARGV[9] ~= "" then
+    rcall("HSET", ARGV[9], "nrjid", nextDelayedJobId)
+end
+return nextDelayedJobId .. "" -- convert to string
+`;
+export const addJobScheduler = {
+    name: 'addJobScheduler',
+    content,
+    keys: 11,
+};
+//# sourceMappingURL=addJobScheduler-11.js.map