bullmq 5.41.5 → 5.41.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/dist/cjs/classes/job-scheduler.js +40 -15
  2. package/dist/cjs/classes/job-scheduler.js.map +1 -1
  3. package/dist/cjs/classes/job.js +9 -12
  4. package/dist/cjs/classes/job.js.map +1 -1
  5. package/dist/cjs/classes/scripts.js +11 -1
  6. package/dist/cjs/classes/scripts.js.map +1 -1
  7. package/dist/cjs/commands/addDelayedJob-6.lua +5 -2
  8. package/dist/cjs/commands/addJobScheduler-10.lua +130 -0
  9. package/dist/cjs/commands/includes/addDelayedJob.lua +2 -7
  10. package/dist/cjs/commands/includes/getOrSetMaxEvents.lua +6 -6
  11. package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +47 -2
  12. package/dist/cjs/commands/moveStalledJobsToWait-9.lua +1 -1
  13. package/dist/cjs/commands/moveToFinished-14.lua +1 -1
  14. package/dist/cjs/scripts/addDelayedJob-6.js +41 -43
  15. package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
  16. package/dist/cjs/scripts/{addJobScheduler-6.js → addJobScheduler-10.js} +208 -42
  17. package/dist/cjs/scripts/addJobScheduler-10.js.map +1 -0
  18. package/dist/cjs/scripts/addParentJob-4.js +6 -6
  19. package/dist/cjs/scripts/addPrioritizedJob-8.js +6 -6
  20. package/dist/cjs/scripts/addStandardJob-8.js +6 -6
  21. package/dist/cjs/scripts/changeDelay-4.js +6 -6
  22. package/dist/cjs/scripts/index.js +1 -1
  23. package/dist/cjs/scripts/index.js.map +1 -1
  24. package/dist/cjs/scripts/moveJobFromActiveToWait-9.js +6 -6
  25. package/dist/cjs/scripts/moveJobsToWait-8.js +6 -6
  26. package/dist/cjs/scripts/moveStalledJobsToWait-9.js +49 -9
  27. package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
  28. package/dist/cjs/scripts/moveToDelayed-8.js +6 -6
  29. package/dist/cjs/scripts/moveToFinished-14.js +49 -9
  30. package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
  31. package/dist/cjs/scripts/removeJob-3.js +6 -6
  32. package/dist/cjs/scripts/reprocessJob-8.js +6 -6
  33. package/dist/cjs/scripts/retryJob-11.js +6 -6
  34. package/dist/cjs/scripts/updateJobScheduler-7.js +8 -41
  35. package/dist/cjs/scripts/updateJobScheduler-7.js.map +1 -1
  36. package/dist/cjs/scripts/updateProgress-3.js +6 -6
  37. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  38. package/dist/cjs/version.js +1 -1
  39. package/dist/esm/classes/job-scheduler.d.ts +1 -0
  40. package/dist/esm/classes/job-scheduler.js +40 -15
  41. package/dist/esm/classes/job-scheduler.js.map +1 -1
  42. package/dist/esm/classes/job.js +9 -12
  43. package/dist/esm/classes/job.js.map +1 -1
  44. package/dist/esm/classes/scripts.d.ts +2 -2
  45. package/dist/esm/classes/scripts.js +11 -1
  46. package/dist/esm/classes/scripts.js.map +1 -1
  47. package/dist/esm/commands/addDelayedJob-6.lua +5 -2
  48. package/dist/esm/commands/addJobScheduler-10.lua +130 -0
  49. package/dist/esm/commands/includes/addDelayedJob.lua +2 -7
  50. package/dist/esm/commands/includes/getOrSetMaxEvents.lua +6 -6
  51. package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +47 -2
  52. package/dist/esm/commands/moveStalledJobsToWait-9.lua +1 -1
  53. package/dist/esm/commands/moveToFinished-14.lua +1 -1
  54. package/dist/esm/scripts/addDelayedJob-6.js +41 -43
  55. package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
  56. package/dist/esm/scripts/{addJobScheduler-6.js → addJobScheduler-10.js} +208 -42
  57. package/dist/esm/scripts/addJobScheduler-10.js.map +1 -0
  58. package/dist/esm/scripts/addParentJob-4.js +6 -6
  59. package/dist/esm/scripts/addPrioritizedJob-8.js +6 -6
  60. package/dist/esm/scripts/addStandardJob-8.js +6 -6
  61. package/dist/esm/scripts/changeDelay-4.js +6 -6
  62. package/dist/esm/scripts/index.d.ts +1 -1
  63. package/dist/esm/scripts/index.js +1 -1
  64. package/dist/esm/scripts/index.js.map +1 -1
  65. package/dist/esm/scripts/moveJobFromActiveToWait-9.js +6 -6
  66. package/dist/esm/scripts/moveJobsToWait-8.js +6 -6
  67. package/dist/esm/scripts/moveStalledJobsToWait-9.js +49 -9
  68. package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
  69. package/dist/esm/scripts/moveToDelayed-8.js +6 -6
  70. package/dist/esm/scripts/moveToFinished-14.js +49 -9
  71. package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
  72. package/dist/esm/scripts/removeJob-3.js +6 -6
  73. package/dist/esm/scripts/reprocessJob-8.js +6 -6
  74. package/dist/esm/scripts/retryJob-11.js +6 -6
  75. package/dist/esm/scripts/updateJobScheduler-7.js +8 -41
  76. package/dist/esm/scripts/updateJobScheduler-7.js.map +1 -1
  77. package/dist/esm/scripts/updateProgress-3.js +6 -6
  78. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  79. package/dist/esm/version.d.ts +1 -1
  80. package/dist/esm/version.js +1 -1
  81. package/package.json +1 -1
  82. package/dist/cjs/commands/addJobScheduler-6.lua +0 -72
  83. package/dist/cjs/scripts/addJobScheduler-6.js.map +0 -1
  84. package/dist/esm/commands/addJobScheduler-6.lua +0 -72
  85. package/dist/esm/scripts/addJobScheduler-6.js.map +0 -1
  86. /package/dist/esm/scripts/{addJobScheduler-6.d.ts → addJobScheduler-10.d.ts} +0 -0
@@ -0,0 +1,130 @@
1
+ --[[
2
+ Adds a job scheduler, i.e. a job factory that creates jobs based on a given schedule (repeat options).
3
+
4
+ Input:
5
+ KEYS[1] 'repeat' key
6
+ KEYS[2] 'delayed' key
7
+ KEYS[3] 'wait' key
8
+ KEYS[4] 'paused' key
9
+ KEYS[5] 'meta' key
10
+ KEYS[6] 'prioritized' key
11
+ KEYS[7] 'marker' key
12
+ KEYS[8] 'id' key
13
+ KEYS[9] 'events' key
14
+ KEYS[10] 'pc' priority counter
15
+
16
+ ARGV[1] next milliseconds
17
+ ARGV[2] msgpacked options
18
+ [1] name
19
+ [2] tz?
20
+ [3] patten?
21
+ [4] endDate?
22
+ [5] every?
23
+ ARGV[3] jobs scheduler id
24
+ ARGV[4] Json stringified template data
25
+ ARGV[5] mspacked template opts
26
+ ARGV[6] msgpacked delayed opts
27
+ ARGV[7] timestamp
28
+ ARGV[8] prefix key
29
+ ARGV[9] producer key
30
+
31
+ Output:
32
+ repeatableKey - OK
33
+ ]]
34
+ local rcall = redis.call
35
+ local repeatKey = KEYS[1]
36
+ local delayedKey = KEYS[2]
37
+ local prioritizedKey = KEYS[6]
38
+
39
+ local nextMillis = ARGV[1]
40
+ local jobSchedulerId = ARGV[3]
41
+ local templateOpts = cmsgpack.unpack(ARGV[5])
42
+ local prefixKey = ARGV[8]
43
+
44
+ -- Includes
45
+ --- @include "includes/addDelayedJob"
46
+ --- @include "includes/addJobWithPriority"
47
+ --- @include "includes/getOrSetMaxEvents"
48
+ --- @include "includes/getTargetQueueList"
49
+ --- @include "includes/isQueuePaused"
50
+ --- @include "includes/removeJob"
51
+ --- @include "includes/storeJob"
52
+ --- @include "includes/storeJobScheduler"
53
+
54
+ -- If we are overriding a repeatable job we must delete the delayed job for
55
+ -- the next iteration.
56
+ local schedulerKey = repeatKey .. ":" .. jobSchedulerId
57
+ local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
58
+ local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
59
+ local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
60
+
61
+ if prevMillis ~= false then
62
+ local currentJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
63
+
64
+ if rcall("EXISTS", nextDelayedJobKey) ~= 1 or currentJobId == nextDelayedJobId then
65
+ if rcall("ZSCORE", delayedKey, currentJobId) ~= false then
66
+ removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
67
+ rcall("ZREM", delayedKey, currentJobId)
68
+ elseif rcall("ZSCORE", prioritizedKey, currentJobId) ~= false then
69
+ removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
70
+ rcall("ZREM", prioritizedKey, currentJobId)
71
+ else
72
+ if isQueuePaused(KEYS[5]) then
73
+ if rcall("LREM", KEYS[4], 1, currentJobId) > 0 then
74
+ removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
75
+ end
76
+ else
77
+ if rcall("LREM", KEYS[3], 1, currentJobId) > 0 then
78
+ removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
79
+ end
80
+ end
81
+ end
82
+ end
83
+ end
84
+
85
+ local schedulerOpts = cmsgpack.unpack(ARGV[2])
86
+ storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, ARGV[4], templateOpts)
87
+
88
+ if rcall("EXISTS", nextDelayedJobKey) ~= 1 then
89
+ local eventsKey = KEYS[9]
90
+ local metaKey = KEYS[5]
91
+ local maxEvents = getOrSetMaxEvents(metaKey)
92
+
93
+ rcall("INCR", KEYS[8])
94
+
95
+ local delayedOpts = cmsgpack.unpack(ARGV[6])
96
+
97
+ local delay, priority = storeJob(eventsKey, nextDelayedJobKey, nextDelayedJobId, schedulerOpts['name'], ARGV[4],
98
+ delayedOpts, ARGV[7], nil, nil, jobSchedulerId)
99
+
100
+ if delay ~= 0 then
101
+ addDelayedJob(nextDelayedJobId, delayedKey, eventsKey,
102
+ ARGV[7], maxEvents, KEYS[7], delay)
103
+ else
104
+ local isPaused = isQueuePaused(KEYS[5])
105
+
106
+ -- Standard or priority add
107
+ if priority == 0 then
108
+ if isPaused then
109
+ -- LIFO or FIFO
110
+ local pushCmd = delayedOpts['lifo'] and 'RPUSH' or 'LPUSH'
111
+ rcall(pushCmd, KEYS[4], nextDelayedJobId)
112
+ else
113
+ -- LIFO or FIFO
114
+ local pushCmd = delayedOpts['lifo'] and 'RPUSH' or 'LPUSH'
115
+ rcall(pushCmd, KEYS[3], nextDelayedJobId)
116
+ end
117
+ else
118
+ -- Priority add
119
+ addJobWithPriority(KEYS[7], KEYS[6], priority, nextDelayedJobId, KEYS[10], isPaused)
120
+ end
121
+ -- Emit waiting event
122
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting", "jobId", nextDelayedJobId)
123
+ end
124
+
125
+ if ARGV[9] ~= "" then
126
+ rcall("HSET", ARGV[9], "nrjid", nextDelayedJobId)
127
+ end
128
+
129
+ return nextDelayedJobId .. "" -- convert to string
130
+ end
@@ -8,13 +8,9 @@
8
8
  -- Includes
9
9
  --- @include "addDelayMarkerIfNeeded"
10
10
  --- @include "getDelayedScore"
11
- --- @include "storeJob"
12
11
 
13
- local function addDelayedJob(jobIdKey, jobId, delayedKey, eventsKey, name, data, opts, timestamp, repeatJobKey,
14
- maxEvents, markerKey, parentKey, parentData)
15
- -- Store the job.
16
- local delay, priority = storeJob(eventsKey, jobIdKey, jobId, name, data,
17
- opts, timestamp, parentKey, parentData, repeatJobKey)
12
+ local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
13
+ maxEvents, markerKey, delay)
18
14
 
19
15
  local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
20
16
 
@@ -25,4 +21,3 @@ local function addDelayedJob(jobIdKey, jobId, delayedKey, eventsKey, name, data,
25
21
  -- mark that a delayed job is available
26
22
  addDelayMarkerIfNeeded(markerKey, delayedKey)
27
23
  end
28
-
@@ -2,10 +2,10 @@
2
2
  Function to get max events value or set by default 10000.
3
3
  ]]
4
4
  local function getOrSetMaxEvents(metaKey)
5
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
6
- if not maxEvents then
7
- maxEvents = 10000
8
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
9
- end
10
- return maxEvents
5
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
6
+ if not maxEvents then
7
+ maxEvents = 10000
8
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
9
+ end
10
+ return maxEvents
11
11
  end
@@ -7,8 +7,11 @@
7
7
  --- @include "removeDeduplicationKeyIfNeeded"
8
8
  --- @include "removeJobsOnFail"
9
9
 
10
- local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
11
- if rcall("ZREM", parentQueueKey .. ":waiting-children", parentId) == 1 then
10
+ local function moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
11
+ local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
12
+ local parentDelayedKey = parentQueueKey .. ":delayed"
13
+ if rcall("ZSCORE", parentWaitingChildrenKey, parentId) ~= false then
14
+ rcall("ZREM", parentWaitingChildrenKey, parentId)
12
15
  local parentQueuePrefix = parentQueueKey .. ":"
13
16
  local parentFailedKey = parentQueueKey .. ":failed"
14
17
  rcall("ZADD", parentFailedKey, timestamp, parentId)
@@ -48,6 +51,48 @@ local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey,
48
51
  local parentRawOpts = jobAttributes[3]
49
52
  local parentOpts = cjson.decode(parentRawOpts)
50
53
 
54
+ removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
55
+ elseif rcall("ZSCORE", parentDelayedKey, parentId) ~= false then
56
+ rcall("ZREM", parentDelayedKey, parentId)
57
+ local parentQueuePrefix = parentQueueKey .. ":"
58
+ local parentFailedKey = parentQueueKey .. ":failed"
59
+ rcall("ZADD", parentFailedKey, timestamp, parentId)
60
+ local failedReason = "child " .. jobIdKey .. " failed"
61
+ rcall("HMSET", parentKey, "failedReason", failedReason, "finishedOn", timestamp)
62
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason",
63
+ failedReason, "prev", "delayed")
64
+
65
+ local jobAttributes = rcall("HMGET", parentKey, "parent", "deid", "opts")
66
+
67
+ removeDeduplicationKeyIfNeeded(parentQueueKey .. ":", jobAttributes[2])
68
+
69
+ if jobAttributes[1] then
70
+ local parentData = cjson.decode(jobAttributes[1])
71
+ if parentData['fpof'] then
72
+ moveParentFromWaitingChildrenToFailed(
73
+ parentData['queueKey'],
74
+ parentData['queueKey'] .. ':' .. parentData['id'],
75
+ parentData['id'],
76
+ parentKey,
77
+ timestamp
78
+ )
79
+ elseif parentData['idof'] or parentData['rdof'] then
80
+ local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
81
+ local grandParentDependenciesSet = grandParentKey .. ":dependencies"
82
+ if rcall("SREM", grandParentDependenciesSet, parentKey) == 1 then
83
+ moveParentToWaitIfNeeded(parentData['queueKey'], grandParentDependenciesSet,
84
+ grandParentKey, parentData['id'], timestamp)
85
+ if parentData['idof'] then
86
+ local grandParentFailedSet = grandParentKey .. ":failed"
87
+ rcall("HSET", grandParentFailedSet, parentKey, failedReason)
88
+ end
89
+ end
90
+ end
91
+ end
92
+
93
+ local parentRawOpts = jobAttributes[3]
94
+ local parentOpts = cjson.decode(parentRawOpts)
95
+
51
96
  removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
52
97
  end
53
98
  end
@@ -95,7 +95,7 @@ if (#stalling > 0) then
95
95
  "failed", "jobId", jobId, 'prev', 'active',
96
96
  'failedReason', failedReason)
97
97
 
98
- if rawParentData ~= false then
98
+ if rawParentData then
99
99
  if opts['fpof'] then
100
100
  local parentData = cjson.decode(rawParentData)
101
101
  moveParentFromWaitingChildrenToFailed(
@@ -103,7 +103,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
103
103
  local parentKey = jobAttributes[1] or ""
104
104
  local parentId = ""
105
105
  local parentQueueKey = ""
106
- if jobAttributes[2] ~= false then
106
+ if jobAttributes[2] then
107
107
  local jsonDecodedParent = cjson.decode(jobAttributes[2])
108
108
  parentId = jsonDecodedParent['id']
109
109
  parentQueueKey = jsonDecodedParent['queueKey']
@@ -103,41 +103,8 @@ local function getDelayedScore(delayedKey, timestamp, delay)
103
103
  end
104
104
  return minScore, delayedTimestamp
105
105
  end
106
- --[[
107
- Function to store a job
108
- ]]
109
- local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
110
- parentKey, parentData, repeatJobKey)
111
- local jsonOpts = cjson.encode(opts)
112
- local delay = opts['delay'] or 0
113
- local priority = opts['priority'] or 0
114
- local debounceId = opts['de'] and opts['de']['id']
115
- local optionalValues = {}
116
- if parentKey ~= nil then
117
- table.insert(optionalValues, "parentKey")
118
- table.insert(optionalValues, parentKey)
119
- table.insert(optionalValues, "parent")
120
- table.insert(optionalValues, parentData)
121
- end
122
- if repeatJobKey ~= nil then
123
- table.insert(optionalValues, "rjk")
124
- table.insert(optionalValues, repeatJobKey)
125
- end
126
- if debounceId then
127
- table.insert(optionalValues, "deid")
128
- table.insert(optionalValues, debounceId)
129
- end
130
- rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
131
- "timestamp", timestamp, "delay", delay, "priority", priority,
132
- unpack(optionalValues))
133
- rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
134
- return delay, priority
135
- end
136
- local function addDelayedJob(jobIdKey, jobId, delayedKey, eventsKey, name, data, opts, timestamp, repeatJobKey,
137
- maxEvents, markerKey, parentKey, parentData)
138
- -- Store the job.
139
- local delay, priority = storeJob(eventsKey, jobIdKey, jobId, name, data,
140
- opts, timestamp, parentKey, parentData, repeatJobKey)
106
+ local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
107
+ maxEvents, markerKey, delay)
141
108
  local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
142
109
  rcall("ZADD", delayedKey, score, jobId)
143
110
  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
@@ -172,12 +139,12 @@ end
172
139
  Function to get max events value or set by default 10000.
173
140
  ]]
174
141
  local function getOrSetMaxEvents(metaKey)
175
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
176
- if not maxEvents then
177
- maxEvents = 10000
178
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
179
- end
180
- return maxEvents
142
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
143
+ if not maxEvents then
144
+ maxEvents = 10000
145
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
146
+ end
147
+ return maxEvents
181
148
  end
182
149
  --[[
183
150
  Function to handle the case when job is duplicated.
@@ -347,6 +314,36 @@ local function handleDuplicatedJob(jobKey, jobId, currentParentKey, currentParen
347
314
  "duplicated", "jobId", jobId)
348
315
  return jobId .. "" -- convert to string
349
316
  end
317
+ --[[
318
+ Function to store a job
319
+ ]]
320
+ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
321
+ parentKey, parentData, repeatJobKey)
322
+ local jsonOpts = cjson.encode(opts)
323
+ local delay = opts['delay'] or 0
324
+ local priority = opts['priority'] or 0
325
+ local debounceId = opts['de'] and opts['de']['id']
326
+ local optionalValues = {}
327
+ if parentKey ~= nil then
328
+ table.insert(optionalValues, "parentKey")
329
+ table.insert(optionalValues, parentKey)
330
+ table.insert(optionalValues, "parent")
331
+ table.insert(optionalValues, parentData)
332
+ end
333
+ if repeatJobKey ~= nil then
334
+ table.insert(optionalValues, "rjk")
335
+ table.insert(optionalValues, repeatJobKey)
336
+ end
337
+ if debounceId then
338
+ table.insert(optionalValues, "deid")
339
+ table.insert(optionalValues, debounceId)
340
+ end
341
+ rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
342
+ "timestamp", timestamp, "delay", delay, "priority", priority,
343
+ unpack(optionalValues))
344
+ rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
345
+ return delay, priority
346
+ end
350
347
  if parentKey ~= nil then
351
348
  if rcall("EXISTS", parentKey) ~= 1 then return -5 end
352
349
  parentData = cjson.encode(parent)
@@ -373,8 +370,9 @@ local deduplicationJobId = deduplicateJob(opts['de'], jobId, deduplicationKey,
373
370
  if deduplicationJobId then
374
371
  return deduplicationJobId
375
372
  end
376
- addDelayedJob(jobIdKey, jobId, delayedKey, eventsKey, args[3], ARGV[2], opts, timestamp, repeatJobKey,
377
- maxEvents, KEYS[1], parentKey, parentData)
373
+ local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
374
+ opts, timestamp, parentKey, parentData, repeatJobKey)
375
+ addDelayedJob(jobId, delayedKey, eventsKey, timestamp, maxEvents, KEYS[1], delay)
378
376
  -- Check if this job is a child of another job, if so add it to the parents dependencies
379
377
  if parentDependenciesKey ~= nil then
380
378
  rcall("SADD", parentDependenciesKey, jobIdKey)
@@ -1 +1 @@
1
- {"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2Xf,CAAC;AACW,QAAA,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyXf,CAAC;AACW,QAAA,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -4,12 +4,16 @@ exports.addJobScheduler = void 0;
4
4
  const content = `--[[
5
5
  Adds a job scheduler, i.e. a job factory that creates jobs based on a given schedule (repeat options).
6
6
  Input:
7
- KEYS[1] 'repeat' key
8
- KEYS[2] 'delayed' key
9
- KEYS[3] 'wait' key
10
- KEYS[4] 'paused' key
11
- KEYS[5] 'meta' key
12
- KEYS[6] 'prioritized' key
7
+ KEYS[1] 'repeat' key
8
+ KEYS[2] 'delayed' key
9
+ KEYS[3] 'wait' key
10
+ KEYS[4] 'paused' key
11
+ KEYS[5] 'meta' key
12
+ KEYS[6] 'prioritized' key
13
+ KEYS[7] 'marker' key
14
+ KEYS[8] 'id' key
15
+ KEYS[9] 'events' key
16
+ KEYS[10] 'pc' priority counter
13
17
  ARGV[1] next milliseconds
14
18
  ARGV[2] msgpacked options
15
19
  [1] name
@@ -20,18 +24,142 @@ const content = `--[[
20
24
  ARGV[3] jobs scheduler id
21
25
  ARGV[4] Json stringified template data
22
26
  ARGV[5] mspacked template opts
23
- ARGV[6] prefix key
27
+ ARGV[6] msgpacked delayed opts
28
+ ARGV[7] timestamp
29
+ ARGV[8] prefix key
30
+ ARGV[9] producer key
24
31
  Output:
25
32
  repeatableKey - OK
26
- ]] local rcall = redis.call
33
+ ]]
34
+ local rcall = redis.call
27
35
  local repeatKey = KEYS[1]
28
36
  local delayedKey = KEYS[2]
29
37
  local prioritizedKey = KEYS[6]
30
38
  local nextMillis = ARGV[1]
31
39
  local jobSchedulerId = ARGV[3]
32
40
  local templateOpts = cmsgpack.unpack(ARGV[5])
33
- local prefixKey = ARGV[6]
41
+ local prefixKey = ARGV[8]
42
+ -- Includes
43
+ --[[
44
+ Adds a delayed job to the queue by doing the following:
45
+ - Creates a new job key with the job data.
46
+ - adds to delayed zset.
47
+ - Emits a global event 'delayed' if the job is delayed.
48
+ ]]
49
+ -- Includes
50
+ --[[
51
+ Add delay marker if needed.
52
+ ]]
53
+ -- Includes
54
+ --[[
55
+ Function to return the next delayed job timestamp.
56
+ ]]
57
+ local function getNextDelayedTimestamp(delayedKey)
58
+ local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
59
+ if #result then
60
+ local nextTimestamp = tonumber(result[2])
61
+ if nextTimestamp ~= nil then
62
+ return nextTimestamp / 0x1000
63
+ end
64
+ end
65
+ end
66
+ local function addDelayMarkerIfNeeded(markerKey, delayedKey)
67
+ local nextTimestamp = getNextDelayedTimestamp(delayedKey)
68
+ if nextTimestamp ~= nil then
69
+ -- Replace the score of the marker with the newest known
70
+ -- next timestamp.
71
+ rcall("ZADD", markerKey, nextTimestamp, "1")
72
+ end
73
+ end
74
+ --[[
75
+ Bake in the job id first 12 bits into the timestamp
76
+ to guarantee correct execution order of delayed jobs
77
+ (up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
78
+ WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
79
+ ]]
80
+ local function getDelayedScore(delayedKey, timestamp, delay)
81
+ local delayedTimestamp = (delay > 0 and (tonumber(timestamp) + delay)) or tonumber(timestamp)
82
+ local minScore = delayedTimestamp * 0x1000
83
+ local maxScore = (delayedTimestamp + 1 ) * 0x1000 - 1
84
+ local result = rcall("ZREVRANGEBYSCORE", delayedKey, maxScore,
85
+ minScore, "WITHSCORES","LIMIT", 0, 1)
86
+ if #result then
87
+ local currentMaxScore = tonumber(result[2])
88
+ if currentMaxScore ~= nil then
89
+ if currentMaxScore >= maxScore then
90
+ return maxScore, delayedTimestamp
91
+ else
92
+ return currentMaxScore + 1, delayedTimestamp
93
+ end
94
+ end
95
+ end
96
+ return minScore, delayedTimestamp
97
+ end
98
+ local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
99
+ maxEvents, markerKey, delay)
100
+ local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
101
+ rcall("ZADD", delayedKey, score, jobId)
102
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
103
+ "jobId", jobId, "delay", delayedTimestamp)
104
+ -- mark that a delayed job is available
105
+ addDelayMarkerIfNeeded(markerKey, delayedKey)
106
+ end
107
+ --[[
108
+ Function to add job considering priority.
109
+ ]]
34
110
  -- Includes
111
+ --[[
112
+ Add marker if needed when a job is available.
113
+ ]]
114
+ local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
115
+ if not isPausedOrMaxed then
116
+ rcall("ZADD", markerKey, 0, "0")
117
+ end
118
+ end
119
+ --[[
120
+ Function to get priority score.
121
+ ]]
122
+ local function getPriorityScore(priority, priorityCounterKey)
123
+ local prioCounter = rcall("INCR", priorityCounterKey)
124
+ return priority * 0x100000000 + prioCounter % 0x100000000
125
+ end
126
+ local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
127
+ isPausedOrMaxed)
128
+ local score = getPriorityScore(priority, priorityCounterKey)
129
+ rcall("ZADD", prioritizedKey, score, jobId)
130
+ addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
131
+ end
132
+ --[[
133
+ Function to get max events value or set by default 10000.
134
+ ]]
135
+ local function getOrSetMaxEvents(metaKey)
136
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
137
+ if not maxEvents then
138
+ maxEvents = 10000
139
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
140
+ end
141
+ return maxEvents
142
+ end
143
+ --[[
144
+ Function to check for the meta.paused key to decide if we are paused or not
145
+ (since an empty list and !EXISTS are not really the same).
146
+ ]]
147
+ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
148
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
149
+ if queueAttributes[1] then
150
+ return pausedKey, true
151
+ else
152
+ if queueAttributes[2] then
153
+ local activeCount = rcall("LLEN", activeKey)
154
+ if activeCount >= tonumber(queueAttributes[2]) then
155
+ return waitKey, true
156
+ else
157
+ return waitKey, false
158
+ end
159
+ end
160
+ end
161
+ return waitKey, false
162
+ end
35
163
  --[[
36
164
  Function to check for the meta.paused key to decide if we are paused or not
37
165
  (since an empty list and !EXISTS are not really the same).
@@ -70,14 +198,6 @@ end
70
198
  Function to add job in target list and add marker if needed.
71
199
  ]]
72
200
  -- Includes
73
- --[[
74
- Add marker if needed when a job is available.
75
- ]]
76
- local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
77
- if not isPausedOrMaxed then
78
- rcall("ZADD", markerKey, 0, "0")
79
- end
80
- end
81
201
  local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
82
202
  rcall(pushCmd, targetKey, jobId)
83
203
  addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
@@ -92,26 +212,6 @@ end
92
212
  local getJobKeyPrefix = function (jobKey, jobId)
93
213
  return string.sub(jobKey, 0, #jobKey - #jobId)
94
214
  end
95
- --[[
96
- Function to check for the meta.paused key to decide if we are paused or not
97
- (since an empty list and !EXISTS are not really the same).
98
- ]]
99
- local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
100
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
101
- if queueAttributes[1] then
102
- return pausedKey, true
103
- else
104
- if queueAttributes[2] then
105
- local activeCount = rcall("LLEN", activeKey)
106
- if activeCount >= tonumber(queueAttributes[2]) then
107
- return waitKey, true
108
- else
109
- return waitKey, false
110
- end
111
- end
112
- end
113
- return waitKey, false
114
- end
115
215
  local function moveParentToWait(parentPrefix, parentId, emitEvent)
116
216
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
117
217
  parentPrefix .. "wait", parentPrefix .. "paused")
@@ -192,6 +292,36 @@ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
192
292
  end
193
293
  removeJobKeys(jobKey)
194
294
  end
295
+ --[[
296
+ Function to store a job
297
+ ]]
298
+ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
299
+ parentKey, parentData, repeatJobKey)
300
+ local jsonOpts = cjson.encode(opts)
301
+ local delay = opts['delay'] or 0
302
+ local priority = opts['priority'] or 0
303
+ local debounceId = opts['de'] and opts['de']['id']
304
+ local optionalValues = {}
305
+ if parentKey ~= nil then
306
+ table.insert(optionalValues, "parentKey")
307
+ table.insert(optionalValues, parentKey)
308
+ table.insert(optionalValues, "parent")
309
+ table.insert(optionalValues, parentData)
310
+ end
311
+ if repeatJobKey ~= nil then
312
+ table.insert(optionalValues, "rjk")
313
+ table.insert(optionalValues, repeatJobKey)
314
+ end
315
+ if debounceId then
316
+ table.insert(optionalValues, "deid")
317
+ table.insert(optionalValues, debounceId)
318
+ end
319
+ rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
320
+ "timestamp", timestamp, "delay", delay, "priority", priority,
321
+ unpack(optionalValues))
322
+ rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
323
+ return delay, priority
324
+ end
195
325
  --[[
196
326
  Function to store a job scheduler
197
327
  ]]
@@ -233,11 +363,11 @@ end
233
363
  -- If we are overriding a repeatable job we must delete the delayed job for
234
364
  -- the next iteration.
235
365
  local schedulerKey = repeatKey .. ":" .. jobSchedulerId
366
+ local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
367
+ local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
236
368
  local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
237
369
  if prevMillis ~= false then
238
370
  local currentJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
239
- local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
240
- local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
241
371
  if rcall("EXISTS", nextDelayedJobKey) ~= 1 or currentJobId == nextDelayedJobId then
242
372
  if rcall("ZSCORE", delayedKey, currentJobId) ~= false then
243
373
  removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
@@ -259,11 +389,47 @@ if prevMillis ~= false then
259
389
  end
260
390
  end
261
391
  local schedulerOpts = cmsgpack.unpack(ARGV[2])
262
- return storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, ARGV[4], templateOpts)
392
+ storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, ARGV[4], templateOpts)
393
+ if rcall("EXISTS", nextDelayedJobKey) ~= 1 then
394
+ local eventsKey = KEYS[9]
395
+ local metaKey = KEYS[5]
396
+ local maxEvents = getOrSetMaxEvents(metaKey)
397
+ rcall("INCR", KEYS[8])
398
+ local delayedOpts = cmsgpack.unpack(ARGV[6])
399
+ local delay, priority = storeJob(eventsKey, nextDelayedJobKey, nextDelayedJobId, schedulerOpts['name'], ARGV[4],
400
+ delayedOpts, ARGV[7], nil, nil, jobSchedulerId)
401
+ if delay ~= 0 then
402
+ addDelayedJob(nextDelayedJobId, delayedKey, eventsKey,
403
+ ARGV[7], maxEvents, KEYS[7], delay)
404
+ else
405
+ local isPaused = isQueuePaused(KEYS[5])
406
+ -- Standard or priority add
407
+ if priority == 0 then
408
+ if isPaused then
409
+ -- LIFO or FIFO
410
+ local pushCmd = delayedOpts['lifo'] and 'RPUSH' or 'LPUSH'
411
+ rcall(pushCmd, KEYS[4], nextDelayedJobId)
412
+ else
413
+ -- LIFO or FIFO
414
+ local pushCmd = delayedOpts['lifo'] and 'RPUSH' or 'LPUSH'
415
+ rcall(pushCmd, KEYS[3], nextDelayedJobId)
416
+ end
417
+ else
418
+ -- Priority add
419
+ addJobWithPriority(KEYS[7], KEYS[6], priority, nextDelayedJobId, KEYS[10], isPaused)
420
+ end
421
+ -- Emit waiting event
422
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting", "jobId", nextDelayedJobId)
423
+ end
424
+ if ARGV[9] ~= "" then
425
+ rcall("HSET", ARGV[9], "nrjid", nextDelayedJobId)
426
+ end
427
+ return nextDelayedJobId .. "" -- convert to string
428
+ end
263
429
  `;
264
430
  exports.addJobScheduler = {
265
431
  name: 'addJobScheduler',
266
432
  content,
267
- keys: 6,
433
+ keys: 10,
268
434
  };
269
- //# sourceMappingURL=addJobScheduler-6.js.map
435
+ //# sourceMappingURL=addJobScheduler-10.js.map