bullmq 5.34.6 → 5.34.7

package/dist/cjs/scripts/addDelayedJob-6.js

@@ -48,6 +48,10 @@ local repeatJobKey = args[9]
 local deduplicationKey = args[10]
 local parentData
 -- Includes
+--[[
+  Add marker if needed when a job is available.
+]]
+-- Includes
 --[[
   Add delay marker if needed.
 ]]
@@ -72,29 +76,6 @@ local function addDelayMarkerIfNeeded(markerKey, delayedKey)
     rcall("ZADD", markerKey, nextTimestamp, "1")
   end
 end
---[[
-  Function to debounce a job.
-]]
-local function deduplicateJob(prefixKey, deduplicationOpts, jobId, deduplicationKey, eventsKey, maxEvents)
-  local deduplicationId = deduplicationOpts and deduplicationOpts['id']
-  if deduplicationId then
-    local ttl = deduplicationOpts['ttl']
-    local deduplicationKeyExists
-    if ttl then
-      deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'PX', ttl, 'NX')
-    else
-      deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'NX')
-    end
-    if deduplicationKeyExists then
-      local currentDebounceJobId = rcall('GET', deduplicationKey)
-      rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
-        "debounced", "jobId", currentDebounceJobId, "debounceId", deduplicationId)
-      rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
-        "deduplicated", "jobId", currentDebounceJobId, "deduplicationId", deduplicationId)
-      return currentDebounceJobId
-    end
-  end
-end
 --[[
   Bake in the job id first 12 bits into the timestamp
   to guarantee correct execution order of delayed jobs
@@ -119,6 +100,71 @@ local function getDelayedScore(delayedKey, timestamp, delay)
   end
   return minScore, delayedTimestamp
 end
+--[[
+  Function to store a job
+]]
+local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
+                        parentKey, parentData, repeatJobKey)
+    local jsonOpts = cjson.encode(opts)
+    local delay = opts['delay'] or 0
+    local priority = opts['priority'] or 0
+    local debounceId = opts['de'] and opts['de']['id']
+    local optionalValues = {}
+    if parentKey ~= nil then
+        table.insert(optionalValues, "parentKey")
+        table.insert(optionalValues, parentKey)
+        table.insert(optionalValues, "parent")
+        table.insert(optionalValues, parentData)
+    end
+    if repeatJobKey ~= nil then
+        table.insert(optionalValues, "rjk")
+        table.insert(optionalValues, repeatJobKey)
+    end
+    if debounceId then
+        table.insert(optionalValues, "deid")
+        table.insert(optionalValues, debounceId)
+    end
+    rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
+          "timestamp", timestamp, "delay", delay, "priority", priority,
+          unpack(optionalValues))
+    rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
+    return delay, priority
+end
+local function addDelayedJob(jobIdKey, jobId, delayedKey, eventsKey, name, data, opts, timestamp, repeatJobKey,
+  maxEvents, markerKey, parentKey, parentData)
+  -- Store the job.
+  local delay, priority = storeJob(eventsKey, jobIdKey, jobId, name, data,
+    opts, timestamp, parentKey, parentData, repeatJobKey)
+  local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
+  rcall("ZADD", delayedKey, score, jobId)
+  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
+    "jobId", jobId, "delay", delayedTimestamp)
+  -- mark that a delayed job is available
+  addDelayMarkerIfNeeded(markerKey, delayedKey)
+end
+--[[
+  Function to debounce a job.
+]]
+local function deduplicateJob(prefixKey, deduplicationOpts, jobId, deduplicationKey, eventsKey, maxEvents)
+  local deduplicationId = deduplicationOpts and deduplicationOpts['id']
+  if deduplicationId then
+    local ttl = deduplicationOpts['ttl']
+    local deduplicationKeyExists
+    if ttl then
+      deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'PX', ttl, 'NX')
+    else
+      deduplicationKeyExists = not rcall('SET', deduplicationKey, jobId, 'NX')
+    end
+    if deduplicationKeyExists then
+      local currentDebounceJobId = rcall('GET', deduplicationKey)
+      rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
+        "debounced", "jobId", currentDebounceJobId, "debounceId", deduplicationId)
+      rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
+        "deduplicated", "jobId", currentDebounceJobId, "deduplicationId", deduplicationId)
+      return currentDebounceJobId
+    end
+  end
+end
 --[[
   Function to get max events value or set by default 10000.
 ]]
@@ -292,36 +338,6 @@ local function handleDuplicatedJob(jobKey, jobId, currentParentKey, currentParen
     "duplicated", "jobId", jobId)
   return jobId .. "" -- convert to string
 end
---[[
-  Function to store a job
-]]
-local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
-                        parentKey, parentData, repeatJobKey)
-    local jsonOpts = cjson.encode(opts)
-    local delay = opts['delay'] or 0
-    local priority = opts['priority'] or 0
-    local debounceId = opts['de'] and opts['de']['id']
-    local optionalValues = {}
-    if parentKey ~= nil then
-        table.insert(optionalValues, "parentKey")
-        table.insert(optionalValues, parentKey)
-        table.insert(optionalValues, "parent")
-        table.insert(optionalValues, parentData)
-    end
-    if repeatJobKey ~= nil then
-        table.insert(optionalValues, "rjk")
-        table.insert(optionalValues, repeatJobKey)
-    end
-    if debounceId then
-        table.insert(optionalValues, "deid")
-        table.insert(optionalValues, debounceId)
-    end
-    rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
-          "timestamp", timestamp, "delay", delay, "priority", priority,
-          unpack(optionalValues))
-    rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
-    return delay, priority
-end
 if parentKey ~= nil then
     if rcall("EXISTS", parentKey) ~= 1 then return -5 end
     parentData = cjson.encode(parent)
@@ -348,17 +364,8 @@ local deduplicationJobId = deduplicateJob(args[1], opts['de'],
 if deduplicationJobId then
   return deduplicationJobId
 end
--- Store the job.
-local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
-                                 opts, timestamp, parentKey, parentData,
-                                 repeatJobKey)
-local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
-rcall("ZADD", delayedKey, score, jobId)
-rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
-      "jobId", jobId, "delay", delayedTimestamp)
--- mark that a delayed job is available
-local markerKey = KEYS[1]
-addDelayMarkerIfNeeded(markerKey, delayedKey)
+addDelayedJob(jobIdKey, jobId, delayedKey, eventsKey, args[3], ARGV[2], opts, timestamp, repeatJobKey,
+  maxEvents, KEYS[1], parentKey, parentData)
 -- Check if this job is a child of another job, if so add it to the parents dependencies
 if parentDependenciesKey ~= nil then
     rcall("SADD", parentDependenciesKey, jobIdKey)

package/dist/cjs/scripts/addDelayedJob-6.js.map

@@ -1 +1 @@
-{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2Wf,CAAC;AACW,QAAA,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
+{"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAkXf,CAAC;AACW,QAAA,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}

package/dist/cjs/scripts/{addJobScheduler-2.js → addJobScheduler-6.js}

@@ -4,8 +4,12 @@ exports.addJobScheduler = void 0;
 const content = `--[[
   Adds a job scheduler, i.e. a job factory that creates jobs based on a given schedule (repeat options).
     Input:
-      KEYS[1] 'repeat' key
-      KEYS[2] 'delayed' key
+      KEYS[1] 'marker',
+      KEYS[2] 'meta'
+      KEYS[3] 'id'
+      KEYS[4] 'delayed'
+      KEYS[5] events stream key
+      KEYS[6] 'repeat' key
       ARGV[1] next milliseconds
       ARGV[2] msgpacked options
             [1]  name
@@ -15,19 +19,128 @@ const content = `--[[
             [5]  every?
       ARGV[3] jobs scheduler id
       ARGV[4] Json stringified template data
-      ARGV[5] mspacked template opts
-      ARGV[6] prefix key
+      ARGV[5] msgpacked template opts
+      ARGV[6] msgpacked delayed opts
+      ARGV[7] timestamp
+      ARGV[8] prefix key
+      ARGV[9] producer key
       Output:
-        repeatableKey  - OK
+        next delayed job id  - OK
 ]]
 local rcall = redis.call
-local repeatKey = KEYS[1]
-local delayedKey = KEYS[2]
+local repeatKey = KEYS[6]
+local delayedKey = KEYS[4]
+local timestamp = ARGV[7]
 local nextMillis = ARGV[1]
 local jobSchedulerId = ARGV[3]
 local templateOpts = cmsgpack.unpack(ARGV[5])
-local prefixKey = ARGV[6]
+local prefixKey = ARGV[8]
 -- Includes
+--[[
+  Add marker if needed when a job is available.
+]]
+-- Includes
+--[[
+  Add delay marker if needed.
+]]
+-- Includes
+--[[
+  Function to return the next delayed job timestamp.
+]]
+local function getNextDelayedTimestamp(delayedKey)
+  local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
+  if #result then
+    local nextTimestamp = tonumber(result[2])
+    if nextTimestamp ~= nil then 
+      return nextTimestamp / 0x1000
+    end
+  end
+end
+local function addDelayMarkerIfNeeded(markerKey, delayedKey)
+  local nextTimestamp = getNextDelayedTimestamp(delayedKey)
+  if nextTimestamp ~= nil then
+    -- Replace the score of the marker with the newest known
+    -- next timestamp.
+    rcall("ZADD", markerKey, nextTimestamp, "1")
+  end
+end
+--[[
+  Bake in the job id first 12 bits into the timestamp
+  to guarantee correct execution order of delayed jobs
+  (up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
+  WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
+]]
+local function getDelayedScore(delayedKey, timestamp, delay)
+  local delayedTimestamp = (delay > 0 and (tonumber(timestamp) + delay)) or tonumber(timestamp)
+  local minScore = delayedTimestamp * 0x1000
+  local maxScore = (delayedTimestamp + 1 ) * 0x1000 - 1
+  local result = rcall("ZREVRANGEBYSCORE", delayedKey, maxScore,
+    minScore, "WITHSCORES","LIMIT", 0, 1)
+  if #result then
+    local currentMaxScore = tonumber(result[2])
+    if currentMaxScore ~= nil then
+      if currentMaxScore >= maxScore then
+        return maxScore, delayedTimestamp
+      else
+        return currentMaxScore + 1, delayedTimestamp
+      end
+    end
+  end
+  return minScore, delayedTimestamp
+end
+--[[
+  Function to store a job
+]]
+local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
+                        parentKey, parentData, repeatJobKey)
+    local jsonOpts = cjson.encode(opts)
+    local delay = opts['delay'] or 0
+    local priority = opts['priority'] or 0
+    local debounceId = opts['de'] and opts['de']['id']
+    local optionalValues = {}
+    if parentKey ~= nil then
+        table.insert(optionalValues, "parentKey")
+        table.insert(optionalValues, parentKey)
+        table.insert(optionalValues, "parent")
+        table.insert(optionalValues, parentData)
+    end
+    if repeatJobKey ~= nil then
+        table.insert(optionalValues, "rjk")
+        table.insert(optionalValues, repeatJobKey)
+    end
+    if debounceId then
+        table.insert(optionalValues, "deid")
+        table.insert(optionalValues, debounceId)
+    end
+    rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
+          "timestamp", timestamp, "delay", delay, "priority", priority,
+          unpack(optionalValues))
+    rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
+    return delay, priority
+end
+local function addDelayedJob(jobIdKey, jobId, delayedKey, eventsKey, name, data, opts, timestamp, repeatJobKey,
+  maxEvents, markerKey, parentKey, parentData)
+  -- Store the job.
+  local delay, priority = storeJob(eventsKey, jobIdKey, jobId, name, data,
+    opts, timestamp, parentKey, parentData, repeatJobKey)
+  local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
+  rcall("ZADD", delayedKey, score, jobId)
+  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
+    "jobId", jobId, "delay", delayedTimestamp)
+  -- mark that a delayed job is available
+  addDelayMarkerIfNeeded(markerKey, delayedKey)
+end
+--[[
+  Function to get max events value or set by default 10000.
+]]
+local function getOrSetMaxEvents(metaKey)
+    local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
+    if not maxEvents then
+        maxEvents = 10000
+        rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
+    end
+    return maxEvents
+end
 --[[
   Function to remove job.
 ]]
@@ -181,9 +294,8 @@ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
   end
   removeJobKeys(jobKey)
 end
-local function storeRepeatableJob(schedulerId, repeatKey, nextMillis, rawOpts, templateData, templateOpts)
+local function storeRepeatableJob(schedulerId, schedulerKey, repeatKey, nextMillis, opts, templateData, templateOpts)
   rcall("ZADD", repeatKey, nextMillis, schedulerId)
-  local opts = cmsgpack.unpack(rawOpts)
   local optionalValues = {}
   if opts['tz'] then
     table.insert(optionalValues, "tz")
@@ -210,16 +322,17 @@ local function storeRepeatableJob(schedulerId, repeatKey, nextMillis, rawOpts, t
     table.insert(optionalValues, "data")
     table.insert(optionalValues, templateData)
   end
-  rcall("HMSET", repeatKey .. ":" .. schedulerId, "name", opts['name'],
+  rcall("HMSET", schedulerKey, "name", opts['name'],
     unpack(optionalValues))
 end
+local schedulerKey = repeatKey .. ":" .. jobSchedulerId
+local nextDelayedJobId =  "repeat:" .. jobSchedulerId .. ":" .. nextMillis
+local nextDelayedJobKey =  schedulerKey .. ":" .. nextMillis
 -- If we are overriding a repeatable job we must delete the delayed job for
 -- the next iteration.
 local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
 if prevMillis ~= false then
   local delayedJobId =  "repeat:" .. jobSchedulerId .. ":" .. prevMillis
-  local nextDelayedJobId =  "repeat:" .. jobSchedulerId .. ":" .. nextMillis
-  local nextDelayedJobKey =  repeatKey .. ":" .. jobSchedulerId .. ":" .. nextMillis
   if rcall("ZSCORE", delayedKey, delayedJobId) ~= false
     and (rcall("EXISTS", nextDelayedJobKey) ~= 1 
     or delayedJobId == nextDelayedJobId) then
@@ -227,11 +340,23 @@ if prevMillis ~= false then
     rcall("ZREM", delayedKey, delayedJobId)
   end
 end
-return storeRepeatableJob(jobSchedulerId, repeatKey, nextMillis, ARGV[2], ARGV[4], templateOpts)
+local schedulerOpts = cmsgpack.unpack(ARGV[2])
+storeRepeatableJob(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, ARGV[4], templateOpts)
+local eventsKey = KEYS[5]
+local metaKey = KEYS[2]
+local maxEvents = getOrSetMaxEvents(metaKey)
+rcall("INCR", KEYS[3])
+local delayedOpts = cmsgpack.unpack(ARGV[6])
+addDelayedJob(nextDelayedJobKey, nextDelayedJobId, delayedKey, eventsKey, schedulerOpts['name'], ARGV[4], delayedOpts,
+  timestamp, jobSchedulerId, maxEvents, KEYS[1], nil, nil)
+if ARGV[9] ~= "" then
+  rcall("HSET", ARGV[9], "nrjid", nextDelayedJobId)
+end
+return nextDelayedJobId .. "" -- convert to string
 `;
 exports.addJobScheduler = {
     name: 'addJobScheduler',
     content,
-    keys: 2,
+    keys: 6,
 };
-//# sourceMappingURL=addJobScheduler-2.js.map
+//# sourceMappingURL=addJobScheduler-6.js.map

package/dist/cjs/scripts/addJobScheduler-6.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"addJobScheduler-6.js","sourceRoot":"","sources":["../../../src/scripts/addJobScheduler-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAgWf,CAAC;AACW,QAAA,eAAe,GAAG;IAC7B,IAAI,EAAE,iBAAiB;IACvB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}

package/dist/cjs/scripts/index.js

@@ -2,7 +2,7 @@
 Object.defineProperty(exports, "__esModule", { value: true });
 const tslib_1 = require("tslib");
 tslib_1.__exportStar(require("./addDelayedJob-6"), exports);
-tslib_1.__exportStar(require("./addJobScheduler-2"), exports);
+tslib_1.__exportStar(require("./addJobScheduler-6"), exports);
 tslib_1.__exportStar(require("./addLog-2"), exports);
 tslib_1.__exportStar(require("./addParentJob-4"), exports);
 tslib_1.__exportStar(require("./addPrioritizedJob-8"), exports);