bullmq 5.41.5 → 5.41.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/dist/cjs/classes/job-scheduler.js +40 -15
  2. package/dist/cjs/classes/job-scheduler.js.map +1 -1
  3. package/dist/cjs/classes/job.js +9 -12
  4. package/dist/cjs/classes/job.js.map +1 -1
  5. package/dist/cjs/classes/scripts.js +11 -1
  6. package/dist/cjs/classes/scripts.js.map +1 -1
  7. package/dist/cjs/commands/addDelayedJob-6.lua +5 -2
  8. package/dist/cjs/commands/addJobScheduler-10.lua +130 -0
  9. package/dist/cjs/commands/includes/addDelayedJob.lua +2 -7
  10. package/dist/cjs/commands/includes/getOrSetMaxEvents.lua +6 -6
  11. package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +47 -2
  12. package/dist/cjs/commands/moveStalledJobsToWait-9.lua +1 -1
  13. package/dist/cjs/commands/moveToFinished-14.lua +1 -1
  14. package/dist/cjs/scripts/addDelayedJob-6.js +41 -43
  15. package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
  16. package/dist/cjs/scripts/{addJobScheduler-6.js → addJobScheduler-10.js} +208 -42
  17. package/dist/cjs/scripts/addJobScheduler-10.js.map +1 -0
  18. package/dist/cjs/scripts/addParentJob-4.js +6 -6
  19. package/dist/cjs/scripts/addPrioritizedJob-8.js +6 -6
  20. package/dist/cjs/scripts/addStandardJob-8.js +6 -6
  21. package/dist/cjs/scripts/changeDelay-4.js +6 -6
  22. package/dist/cjs/scripts/index.js +1 -1
  23. package/dist/cjs/scripts/index.js.map +1 -1
  24. package/dist/cjs/scripts/moveJobFromActiveToWait-9.js +6 -6
  25. package/dist/cjs/scripts/moveJobsToWait-8.js +6 -6
  26. package/dist/cjs/scripts/moveStalledJobsToWait-9.js +49 -9
  27. package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
  28. package/dist/cjs/scripts/moveToDelayed-8.js +6 -6
  29. package/dist/cjs/scripts/moveToFinished-14.js +49 -9
  30. package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
  31. package/dist/cjs/scripts/removeJob-3.js +6 -6
  32. package/dist/cjs/scripts/reprocessJob-8.js +6 -6
  33. package/dist/cjs/scripts/retryJob-11.js +6 -6
  34. package/dist/cjs/scripts/updateJobScheduler-7.js +8 -41
  35. package/dist/cjs/scripts/updateJobScheduler-7.js.map +1 -1
  36. package/dist/cjs/scripts/updateProgress-3.js +6 -6
  37. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  38. package/dist/cjs/version.js +1 -1
  39. package/dist/esm/classes/job-scheduler.d.ts +1 -0
  40. package/dist/esm/classes/job-scheduler.js +40 -15
  41. package/dist/esm/classes/job-scheduler.js.map +1 -1
  42. package/dist/esm/classes/job.js +9 -12
  43. package/dist/esm/classes/job.js.map +1 -1
  44. package/dist/esm/classes/scripts.d.ts +2 -2
  45. package/dist/esm/classes/scripts.js +11 -1
  46. package/dist/esm/classes/scripts.js.map +1 -1
  47. package/dist/esm/commands/addDelayedJob-6.lua +5 -2
  48. package/dist/esm/commands/addJobScheduler-10.lua +130 -0
  49. package/dist/esm/commands/includes/addDelayedJob.lua +2 -7
  50. package/dist/esm/commands/includes/getOrSetMaxEvents.lua +6 -6
  51. package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +47 -2
  52. package/dist/esm/commands/moveStalledJobsToWait-9.lua +1 -1
  53. package/dist/esm/commands/moveToFinished-14.lua +1 -1
  54. package/dist/esm/scripts/addDelayedJob-6.js +41 -43
  55. package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
  56. package/dist/esm/scripts/{addJobScheduler-6.js → addJobScheduler-10.js} +208 -42
  57. package/dist/esm/scripts/addJobScheduler-10.js.map +1 -0
  58. package/dist/esm/scripts/addParentJob-4.js +6 -6
  59. package/dist/esm/scripts/addPrioritizedJob-8.js +6 -6
  60. package/dist/esm/scripts/addStandardJob-8.js +6 -6
  61. package/dist/esm/scripts/changeDelay-4.js +6 -6
  62. package/dist/esm/scripts/index.d.ts +1 -1
  63. package/dist/esm/scripts/index.js +1 -1
  64. package/dist/esm/scripts/index.js.map +1 -1
  65. package/dist/esm/scripts/moveJobFromActiveToWait-9.js +6 -6
  66. package/dist/esm/scripts/moveJobsToWait-8.js +6 -6
  67. package/dist/esm/scripts/moveStalledJobsToWait-9.js +49 -9
  68. package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
  69. package/dist/esm/scripts/moveToDelayed-8.js +6 -6
  70. package/dist/esm/scripts/moveToFinished-14.js +49 -9
  71. package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
  72. package/dist/esm/scripts/removeJob-3.js +6 -6
  73. package/dist/esm/scripts/reprocessJob-8.js +6 -6
  74. package/dist/esm/scripts/retryJob-11.js +6 -6
  75. package/dist/esm/scripts/updateJobScheduler-7.js +8 -41
  76. package/dist/esm/scripts/updateJobScheduler-7.js.map +1 -1
  77. package/dist/esm/scripts/updateProgress-3.js +6 -6
  78. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  79. package/dist/esm/version.d.ts +1 -1
  80. package/dist/esm/version.js +1 -1
  81. package/package.json +1 -1
  82. package/dist/cjs/commands/addJobScheduler-6.lua +0 -72
  83. package/dist/cjs/scripts/addJobScheduler-6.js.map +0 -1
  84. package/dist/esm/commands/addJobScheduler-6.lua +0 -72
  85. package/dist/esm/scripts/addJobScheduler-6.js.map +0 -1
  86. /package/dist/esm/scripts/{addJobScheduler-6.d.ts → addJobScheduler-10.d.ts} +0 -0
@@ -100,41 +100,8 @@ local function getDelayedScore(delayedKey, timestamp, delay)
100
100
  end
101
101
  return minScore, delayedTimestamp
102
102
  end
103
- --[[
104
- Function to store a job
105
- ]]
106
- local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
107
- parentKey, parentData, repeatJobKey)
108
- local jsonOpts = cjson.encode(opts)
109
- local delay = opts['delay'] or 0
110
- local priority = opts['priority'] or 0
111
- local debounceId = opts['de'] and opts['de']['id']
112
- local optionalValues = {}
113
- if parentKey ~= nil then
114
- table.insert(optionalValues, "parentKey")
115
- table.insert(optionalValues, parentKey)
116
- table.insert(optionalValues, "parent")
117
- table.insert(optionalValues, parentData)
118
- end
119
- if repeatJobKey ~= nil then
120
- table.insert(optionalValues, "rjk")
121
- table.insert(optionalValues, repeatJobKey)
122
- end
123
- if debounceId then
124
- table.insert(optionalValues, "deid")
125
- table.insert(optionalValues, debounceId)
126
- end
127
- rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
128
- "timestamp", timestamp, "delay", delay, "priority", priority,
129
- unpack(optionalValues))
130
- rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
131
- return delay, priority
132
- end
133
- local function addDelayedJob(jobIdKey, jobId, delayedKey, eventsKey, name, data, opts, timestamp, repeatJobKey,
134
- maxEvents, markerKey, parentKey, parentData)
135
- -- Store the job.
136
- local delay, priority = storeJob(eventsKey, jobIdKey, jobId, name, data,
137
- opts, timestamp, parentKey, parentData, repeatJobKey)
103
+ local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
104
+ maxEvents, markerKey, delay)
138
105
  local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
139
106
  rcall("ZADD", delayedKey, score, jobId)
140
107
  rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
@@ -169,12 +136,12 @@ end
169
136
  Function to get max events value or set by default 10000.
170
137
  ]]
171
138
  local function getOrSetMaxEvents(metaKey)
172
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
173
- if not maxEvents then
174
- maxEvents = 10000
175
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
176
- end
177
- return maxEvents
139
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
140
+ if not maxEvents then
141
+ maxEvents = 10000
142
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
143
+ end
144
+ return maxEvents
178
145
  end
179
146
  --[[
180
147
  Function to handle the case when job is duplicated.
@@ -344,6 +311,36 @@ local function handleDuplicatedJob(jobKey, jobId, currentParentKey, currentParen
344
311
  "duplicated", "jobId", jobId)
345
312
  return jobId .. "" -- convert to string
346
313
  end
314
+ --[[
315
+ Function to store a job
316
+ ]]
317
+ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
318
+ parentKey, parentData, repeatJobKey)
319
+ local jsonOpts = cjson.encode(opts)
320
+ local delay = opts['delay'] or 0
321
+ local priority = opts['priority'] or 0
322
+ local debounceId = opts['de'] and opts['de']['id']
323
+ local optionalValues = {}
324
+ if parentKey ~= nil then
325
+ table.insert(optionalValues, "parentKey")
326
+ table.insert(optionalValues, parentKey)
327
+ table.insert(optionalValues, "parent")
328
+ table.insert(optionalValues, parentData)
329
+ end
330
+ if repeatJobKey ~= nil then
331
+ table.insert(optionalValues, "rjk")
332
+ table.insert(optionalValues, repeatJobKey)
333
+ end
334
+ if debounceId then
335
+ table.insert(optionalValues, "deid")
336
+ table.insert(optionalValues, debounceId)
337
+ end
338
+ rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
339
+ "timestamp", timestamp, "delay", delay, "priority", priority,
340
+ unpack(optionalValues))
341
+ rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
342
+ return delay, priority
343
+ end
347
344
  if parentKey ~= nil then
348
345
  if rcall("EXISTS", parentKey) ~= 1 then return -5 end
349
346
  parentData = cjson.encode(parent)
@@ -370,8 +367,9 @@ local deduplicationJobId = deduplicateJob(opts['de'], jobId, deduplicationKey,
370
367
  if deduplicationJobId then
371
368
  return deduplicationJobId
372
369
  end
373
- addDelayedJob(jobIdKey, jobId, delayedKey, eventsKey, args[3], ARGV[2], opts, timestamp, repeatJobKey,
374
- maxEvents, KEYS[1], parentKey, parentData)
370
+ local delay, priority = storeJob(eventsKey, jobIdKey, jobId, args[3], ARGV[2],
371
+ opts, timestamp, parentKey, parentData, repeatJobKey)
372
+ addDelayedJob(jobId, delayedKey, eventsKey, timestamp, maxEvents, KEYS[1], delay)
375
373
  -- Check if this job is a child of another job, if so add it to the parents dependencies
376
374
  if parentDependenciesKey ~= nil then
377
375
  rcall("SADD", parentDependenciesKey, jobIdKey)
@@ -1 +1 @@
1
- {"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA2Xf,CAAC;AACF,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"addDelayedJob-6.js","sourceRoot":"","sources":["../../../src/scripts/addDelayedJob-6.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyXf,CAAC;AACF,MAAM,CAAC,MAAM,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -1,12 +1,16 @@
1
1
  const content = `--[[
2
2
  Adds a job scheduler, i.e. a job factory that creates jobs based on a given schedule (repeat options).
3
3
  Input:
4
- KEYS[1] 'repeat' key
5
- KEYS[2] 'delayed' key
6
- KEYS[3] 'wait' key
7
- KEYS[4] 'paused' key
8
- KEYS[5] 'meta' key
9
- KEYS[6] 'prioritized' key
4
+ KEYS[1] 'repeat' key
5
+ KEYS[2] 'delayed' key
6
+ KEYS[3] 'wait' key
7
+ KEYS[4] 'paused' key
8
+ KEYS[5] 'meta' key
9
+ KEYS[6] 'prioritized' key
10
+ KEYS[7] 'marker' key
11
+ KEYS[8] 'id' key
12
+ KEYS[9] 'events' key
13
+ KEYS[10] 'pc' priority counter
10
14
  ARGV[1] next milliseconds
11
15
  ARGV[2] msgpacked options
12
16
  [1] name
@@ -17,18 +21,142 @@ const content = `--[[
17
21
  ARGV[3] jobs scheduler id
18
22
  ARGV[4] Json stringified template data
19
23
  ARGV[5] mspacked template opts
20
- ARGV[6] prefix key
24
+ ARGV[6] msgpacked delayed opts
25
+ ARGV[7] timestamp
26
+ ARGV[8] prefix key
27
+ ARGV[9] producer key
21
28
  Output:
22
29
  repeatableKey - OK
23
- ]] local rcall = redis.call
30
+ ]]
31
+ local rcall = redis.call
24
32
  local repeatKey = KEYS[1]
25
33
  local delayedKey = KEYS[2]
26
34
  local prioritizedKey = KEYS[6]
27
35
  local nextMillis = ARGV[1]
28
36
  local jobSchedulerId = ARGV[3]
29
37
  local templateOpts = cmsgpack.unpack(ARGV[5])
30
- local prefixKey = ARGV[6]
38
+ local prefixKey = ARGV[8]
39
+ -- Includes
40
+ --[[
41
+ Adds a delayed job to the queue by doing the following:
42
+ - Creates a new job key with the job data.
43
+ - adds to delayed zset.
44
+ - Emits a global event 'delayed' if the job is delayed.
45
+ ]]
46
+ -- Includes
47
+ --[[
48
+ Add delay marker if needed.
49
+ ]]
50
+ -- Includes
51
+ --[[
52
+ Function to return the next delayed job timestamp.
53
+ ]]
54
+ local function getNextDelayedTimestamp(delayedKey)
55
+ local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
56
+ if #result then
57
+ local nextTimestamp = tonumber(result[2])
58
+ if nextTimestamp ~= nil then
59
+ return nextTimestamp / 0x1000
60
+ end
61
+ end
62
+ end
63
+ local function addDelayMarkerIfNeeded(markerKey, delayedKey)
64
+ local nextTimestamp = getNextDelayedTimestamp(delayedKey)
65
+ if nextTimestamp ~= nil then
66
+ -- Replace the score of the marker with the newest known
67
+ -- next timestamp.
68
+ rcall("ZADD", markerKey, nextTimestamp, "1")
69
+ end
70
+ end
71
+ --[[
72
+ Bake in the job id first 12 bits into the timestamp
73
+ to guarantee correct execution order of delayed jobs
74
+ (up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
75
+ WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
76
+ ]]
77
+ local function getDelayedScore(delayedKey, timestamp, delay)
78
+ local delayedTimestamp = (delay > 0 and (tonumber(timestamp) + delay)) or tonumber(timestamp)
79
+ local minScore = delayedTimestamp * 0x1000
80
+ local maxScore = (delayedTimestamp + 1 ) * 0x1000 - 1
81
+ local result = rcall("ZREVRANGEBYSCORE", delayedKey, maxScore,
82
+ minScore, "WITHSCORES","LIMIT", 0, 1)
83
+ if #result then
84
+ local currentMaxScore = tonumber(result[2])
85
+ if currentMaxScore ~= nil then
86
+ if currentMaxScore >= maxScore then
87
+ return maxScore, delayedTimestamp
88
+ else
89
+ return currentMaxScore + 1, delayedTimestamp
90
+ end
91
+ end
92
+ end
93
+ return minScore, delayedTimestamp
94
+ end
95
+ local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
96
+ maxEvents, markerKey, delay)
97
+ local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
98
+ rcall("ZADD", delayedKey, score, jobId)
99
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
100
+ "jobId", jobId, "delay", delayedTimestamp)
101
+ -- mark that a delayed job is available
102
+ addDelayMarkerIfNeeded(markerKey, delayedKey)
103
+ end
104
+ --[[
105
+ Function to add job considering priority.
106
+ ]]
31
107
  -- Includes
108
+ --[[
109
+ Add marker if needed when a job is available.
110
+ ]]
111
+ local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
112
+ if not isPausedOrMaxed then
113
+ rcall("ZADD", markerKey, 0, "0")
114
+ end
115
+ end
116
+ --[[
117
+ Function to get priority score.
118
+ ]]
119
+ local function getPriorityScore(priority, priorityCounterKey)
120
+ local prioCounter = rcall("INCR", priorityCounterKey)
121
+ return priority * 0x100000000 + prioCounter % 0x100000000
122
+ end
123
+ local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
124
+ isPausedOrMaxed)
125
+ local score = getPriorityScore(priority, priorityCounterKey)
126
+ rcall("ZADD", prioritizedKey, score, jobId)
127
+ addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
128
+ end
129
+ --[[
130
+ Function to get max events value or set by default 10000.
131
+ ]]
132
+ local function getOrSetMaxEvents(metaKey)
133
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
134
+ if not maxEvents then
135
+ maxEvents = 10000
136
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
137
+ end
138
+ return maxEvents
139
+ end
140
+ --[[
141
+ Function to check for the meta.paused key to decide if we are paused or not
142
+ (since an empty list and !EXISTS are not really the same).
143
+ ]]
144
+ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
145
+ local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
146
+ if queueAttributes[1] then
147
+ return pausedKey, true
148
+ else
149
+ if queueAttributes[2] then
150
+ local activeCount = rcall("LLEN", activeKey)
151
+ if activeCount >= tonumber(queueAttributes[2]) then
152
+ return waitKey, true
153
+ else
154
+ return waitKey, false
155
+ end
156
+ end
157
+ end
158
+ return waitKey, false
159
+ end
32
160
  --[[
33
161
  Function to check for the meta.paused key to decide if we are paused or not
34
162
  (since an empty list and !EXISTS are not really the same).
@@ -67,14 +195,6 @@ end
67
195
  Function to add job in target list and add marker if needed.
68
196
  ]]
69
197
  -- Includes
70
- --[[
71
- Add marker if needed when a job is available.
72
- ]]
73
- local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
74
- if not isPausedOrMaxed then
75
- rcall("ZADD", markerKey, 0, "0")
76
- end
77
- end
78
198
  local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
79
199
  rcall(pushCmd, targetKey, jobId)
80
200
  addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
@@ -89,26 +209,6 @@ end
89
209
  local getJobKeyPrefix = function (jobKey, jobId)
90
210
  return string.sub(jobKey, 0, #jobKey - #jobId)
91
211
  end
92
- --[[
93
- Function to check for the meta.paused key to decide if we are paused or not
94
- (since an empty list and !EXISTS are not really the same).
95
- ]]
96
- local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
97
- local queueAttributes = rcall("HMGET", queueMetaKey, "paused", "concurrency")
98
- if queueAttributes[1] then
99
- return pausedKey, true
100
- else
101
- if queueAttributes[2] then
102
- local activeCount = rcall("LLEN", activeKey)
103
- if activeCount >= tonumber(queueAttributes[2]) then
104
- return waitKey, true
105
- else
106
- return waitKey, false
107
- end
108
- end
109
- end
110
- return waitKey, false
111
- end
112
212
  local function moveParentToWait(parentPrefix, parentId, emitEvent)
113
213
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
114
214
  parentPrefix .. "wait", parentPrefix .. "paused")
@@ -189,6 +289,36 @@ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
189
289
  end
190
290
  removeJobKeys(jobKey)
191
291
  end
292
+ --[[
293
+ Function to store a job
294
+ ]]
295
+ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
296
+ parentKey, parentData, repeatJobKey)
297
+ local jsonOpts = cjson.encode(opts)
298
+ local delay = opts['delay'] or 0
299
+ local priority = opts['priority'] or 0
300
+ local debounceId = opts['de'] and opts['de']['id']
301
+ local optionalValues = {}
302
+ if parentKey ~= nil then
303
+ table.insert(optionalValues, "parentKey")
304
+ table.insert(optionalValues, parentKey)
305
+ table.insert(optionalValues, "parent")
306
+ table.insert(optionalValues, parentData)
307
+ end
308
+ if repeatJobKey ~= nil then
309
+ table.insert(optionalValues, "rjk")
310
+ table.insert(optionalValues, repeatJobKey)
311
+ end
312
+ if debounceId then
313
+ table.insert(optionalValues, "deid")
314
+ table.insert(optionalValues, debounceId)
315
+ end
316
+ rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
317
+ "timestamp", timestamp, "delay", delay, "priority", priority,
318
+ unpack(optionalValues))
319
+ rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
320
+ return delay, priority
321
+ end
192
322
  --[[
193
323
  Function to store a job scheduler
194
324
  ]]
@@ -230,11 +360,11 @@ end
230
360
  -- If we are overriding a repeatable job we must delete the delayed job for
231
361
  -- the next iteration.
232
362
  local schedulerKey = repeatKey .. ":" .. jobSchedulerId
363
+ local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
364
+ local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
233
365
  local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
234
366
  if prevMillis ~= false then
235
367
  local currentJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
236
- local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
237
- local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
238
368
  if rcall("EXISTS", nextDelayedJobKey) ~= 1 or currentJobId == nextDelayedJobId then
239
369
  if rcall("ZSCORE", delayedKey, currentJobId) ~= false then
240
370
  removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
@@ -256,11 +386,47 @@ if prevMillis ~= false then
256
386
  end
257
387
  end
258
388
  local schedulerOpts = cmsgpack.unpack(ARGV[2])
259
- return storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, ARGV[4], templateOpts)
389
+ storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, ARGV[4], templateOpts)
390
+ if rcall("EXISTS", nextDelayedJobKey) ~= 1 then
391
+ local eventsKey = KEYS[9]
392
+ local metaKey = KEYS[5]
393
+ local maxEvents = getOrSetMaxEvents(metaKey)
394
+ rcall("INCR", KEYS[8])
395
+ local delayedOpts = cmsgpack.unpack(ARGV[6])
396
+ local delay, priority = storeJob(eventsKey, nextDelayedJobKey, nextDelayedJobId, schedulerOpts['name'], ARGV[4],
397
+ delayedOpts, ARGV[7], nil, nil, jobSchedulerId)
398
+ if delay ~= 0 then
399
+ addDelayedJob(nextDelayedJobId, delayedKey, eventsKey,
400
+ ARGV[7], maxEvents, KEYS[7], delay)
401
+ else
402
+ local isPaused = isQueuePaused(KEYS[5])
403
+ -- Standard or priority add
404
+ if priority == 0 then
405
+ if isPaused then
406
+ -- LIFO or FIFO
407
+ local pushCmd = delayedOpts['lifo'] and 'RPUSH' or 'LPUSH'
408
+ rcall(pushCmd, KEYS[4], nextDelayedJobId)
409
+ else
410
+ -- LIFO or FIFO
411
+ local pushCmd = delayedOpts['lifo'] and 'RPUSH' or 'LPUSH'
412
+ rcall(pushCmd, KEYS[3], nextDelayedJobId)
413
+ end
414
+ else
415
+ -- Priority add
416
+ addJobWithPriority(KEYS[7], KEYS[6], priority, nextDelayedJobId, KEYS[10], isPaused)
417
+ end
418
+ -- Emit waiting event
419
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting", "jobId", nextDelayedJobId)
420
+ end
421
+ if ARGV[9] ~= "" then
422
+ rcall("HSET", ARGV[9], "nrjid", nextDelayedJobId)
423
+ end
424
+ return nextDelayedJobId .. "" -- convert to string
425
+ end
260
426
  `;
261
427
  export const addJobScheduler = {
262
428
  name: 'addJobScheduler',
263
429
  content,
264
- keys: 6,
430
+ keys: 10,
265
431
  };
266
- //# sourceMappingURL=addJobScheduler-6.js.map
432
+ //# sourceMappingURL=addJobScheduler-10.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"addJobScheduler-10.js","sourceRoot":"","sources":["../../../src/scripts/addJobScheduler-10.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyaf,CAAC;AACF,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,IAAI,EAAE,iBAAiB;IACvB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
@@ -68,12 +68,12 @@ end
68
68
  Function to get max events value or set by default 10000.
69
69
  ]]
70
70
  local function getOrSetMaxEvents(metaKey)
71
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
72
- if not maxEvents then
73
- maxEvents = 10000
74
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
75
- end
76
- return maxEvents
71
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
72
+ if not maxEvents then
73
+ maxEvents = 10000
74
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
75
+ end
76
+ return maxEvents
77
77
  end
78
78
  --[[
79
79
  Function to handle the case when job is duplicated.
@@ -130,12 +130,12 @@ end
130
130
  Function to get max events value or set by default 10000.
131
131
  ]]
132
132
  local function getOrSetMaxEvents(metaKey)
133
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
134
- if not maxEvents then
135
- maxEvents = 10000
136
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
137
- end
138
- return maxEvents
133
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
134
+ if not maxEvents then
135
+ maxEvents = 10000
136
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
137
+ end
138
+ return maxEvents
139
139
  end
140
140
  --[[
141
141
  Function to handle the case when job is duplicated.
@@ -94,12 +94,12 @@ end
94
94
  Function to get max events value or set by default 10000.
95
95
  ]]
96
96
  local function getOrSetMaxEvents(metaKey)
97
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
98
- if not maxEvents then
99
- maxEvents = 10000
100
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
101
- end
102
- return maxEvents
97
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
98
+ if not maxEvents then
99
+ maxEvents = 10000
100
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
101
+ end
102
+ return maxEvents
103
103
  end
104
104
  --[[
105
105
  Function to check for the meta.paused key to decide if we are paused or not
@@ -70,12 +70,12 @@ end
70
70
  Function to get max events value or set by default 10000.
71
71
  ]]
72
72
  local function getOrSetMaxEvents(metaKey)
73
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
74
- if not maxEvents then
75
- maxEvents = 10000
76
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
77
- end
78
- return maxEvents
73
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
74
+ if not maxEvents then
75
+ maxEvents = 10000
76
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
77
+ end
78
+ return maxEvents
79
79
  end
80
80
  if rcall("EXISTS", ARGV[4]) == 1 then
81
81
  local jobId = ARGV[3]
@@ -1,5 +1,5 @@
1
1
  export * from './addDelayedJob-6';
2
- export * from './addJobScheduler-6';
2
+ export * from './addJobScheduler-10';
3
3
  export * from './addLog-2';
4
4
  export * from './addParentJob-4';
5
5
  export * from './addPrioritizedJob-8';
@@ -1,5 +1,5 @@
1
1
  export * from './addDelayedJob-6';
2
- export * from './addJobScheduler-6';
2
+ export * from './addJobScheduler-10';
3
3
  export * from './addLog-2';
4
4
  export * from './addParentJob-4';
5
5
  export * from './addPrioritizedJob-8';
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/scripts/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,YAAY,CAAC;AAC3B,cAAc,kBAAkB,CAAC;AACjC,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AACnC,cAAc,oBAAoB,CAAC;AACnC,cAAc,WAAW,CAAC;AAC1B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,eAAe,CAAC;AAC9B,cAAc,0BAA0B,CAAC;AACzC,cAAc,qBAAqB,CAAC;AACpC,cAAc,eAAe,CAAC;AAC9B,cAAc,qBAAqB,CAAC;AACpC,cAAc,cAAc,CAAC;AAC7B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,aAAa,CAAC;AAC5B,cAAc,6BAA6B,CAAC;AAC5C,cAAc,oBAAoB,CAAC;AACnC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,mBAAmB,CAAC;AAClC,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,gBAAgB,CAAC;AAC/B,cAAc,cAAc,CAAC;AAC7B,cAAc,WAAW,CAAC;AAC1B,cAAc,aAAa,CAAC;AAC5B,cAAc,iBAAiB,CAAC;AAChC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,eAAe,CAAC;AAC9B,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,kBAAkB,CAAC;AACjC,cAAc,eAAe,CAAC;AAC9B,cAAc,oBAAoB,CAAC;AACnC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,+BAA+B,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/scripts/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,sBAAsB,CAAC;AACrC,cAAc,YAAY,CAAC;AAC3B,cAAc,kBAAkB,CAAC;AACjC,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AACnC,cAAc,oBAAoB,CAAC;AACnC,cAAc,WAAW,CAAC;AAC1B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,eAAe,CAAC;AAC9B,cAAc,0BAA0B,CAAC;AACzC,cAAc,qBAAqB,CAAC;AACpC,cAAc,eAAe,CAAC;AAC9B,cAAc,qBAAqB,CAAC;AACpC,cAAc,cAAc,CAAC;AAC7B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,aAAa,CAAC;AAC5B,cAAc,6BAA6B,CAAC;AAC5C,cAAc,oBAAoB,CAAC;AACnC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,mBAAmB,CAAC;AAClC,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,gBAAgB,CAAC;AAC/B,cAAc,cAAc,CAAC;AAC7B,cAAc,WAAW,CAAC;AAC1B,cAAc,aAAa,CAAC;AAC5B,cAAc,iBAAiB,CAAC;AAChC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,eAAe,CAAC;AAC9B,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,kBAAkB,CAAC;AACjC,cAAc,eAAe,CAAC;AAC9B,cAAc,oBAAoB,CAAC;AACnC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,+BAA+B,CAAC"}
@@ -45,12 +45,12 @@ end
45
45
  Function to get max events value or set by default 10000.
46
46
  ]]
47
47
  local function getOrSetMaxEvents(metaKey)
48
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
49
- if not maxEvents then
50
- maxEvents = 10000
51
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
52
- end
53
- return maxEvents
48
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
49
+ if not maxEvents then
50
+ maxEvents = 10000
51
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
52
+ end
53
+ return maxEvents
54
54
  end
55
55
  --[[
56
56
  Function to check for the meta.paused key to decide if we are paused or not
@@ -49,12 +49,12 @@ end
49
49
  Function to get max events value or set by default 10000.
50
50
  ]]
51
51
  local function getOrSetMaxEvents(metaKey)
52
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
53
- if not maxEvents then
54
- maxEvents = 10000
55
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
56
- end
57
- return maxEvents
52
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
53
+ if not maxEvents then
54
+ maxEvents = 10000
55
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
56
+ end
57
+ return maxEvents
58
58
  end
59
59
  --[[
60
60
  Function to check for the meta.paused key to decide if we are paused or not
@@ -358,8 +358,11 @@ local function removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestam
358
358
  end
359
359
  end
360
360
  end
361
- local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
362
- if rcall("ZREM", parentQueueKey .. ":waiting-children", parentId) == 1 then
361
+ local function moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
362
+ local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
363
+ local parentDelayedKey = parentQueueKey .. ":delayed"
364
+ if rcall("ZSCORE", parentWaitingChildrenKey, parentId) ~= false then
365
+ rcall("ZREM", parentWaitingChildrenKey, parentId)
363
366
  local parentQueuePrefix = parentQueueKey .. ":"
364
367
  local parentFailedKey = parentQueueKey .. ":failed"
365
368
  rcall("ZADD", parentFailedKey, timestamp, parentId)
@@ -395,6 +398,43 @@ local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey,
395
398
  local parentRawOpts = jobAttributes[3]
396
399
  local parentOpts = cjson.decode(parentRawOpts)
397
400
  removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
401
+ elseif rcall("ZSCORE", parentDelayedKey, parentId) ~= false then
402
+ rcall("ZREM", parentDelayedKey, parentId)
403
+ local parentQueuePrefix = parentQueueKey .. ":"
404
+ local parentFailedKey = parentQueueKey .. ":failed"
405
+ rcall("ZADD", parentFailedKey, timestamp, parentId)
406
+ local failedReason = "child " .. jobIdKey .. " failed"
407
+ rcall("HMSET", parentKey, "failedReason", failedReason, "finishedOn", timestamp)
408
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason",
409
+ failedReason, "prev", "delayed")
410
+ local jobAttributes = rcall("HMGET", parentKey, "parent", "deid", "opts")
411
+ removeDeduplicationKeyIfNeeded(parentQueueKey .. ":", jobAttributes[2])
412
+ if jobAttributes[1] then
413
+ local parentData = cjson.decode(jobAttributes[1])
414
+ if parentData['fpof'] then
415
+ moveParentFromWaitingChildrenToFailed(
416
+ parentData['queueKey'],
417
+ parentData['queueKey'] .. ':' .. parentData['id'],
418
+ parentData['id'],
419
+ parentKey,
420
+ timestamp
421
+ )
422
+ elseif parentData['idof'] or parentData['rdof'] then
423
+ local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
424
+ local grandParentDependenciesSet = grandParentKey .. ":dependencies"
425
+ if rcall("SREM", grandParentDependenciesSet, parentKey) == 1 then
426
+ moveParentToWaitIfNeeded(parentData['queueKey'], grandParentDependenciesSet,
427
+ grandParentKey, parentData['id'], timestamp)
428
+ if parentData['idof'] then
429
+ local grandParentFailedSet = grandParentKey .. ":failed"
430
+ rcall("HSET", grandParentFailedSet, parentKey, failedReason)
431
+ end
432
+ end
433
+ end
434
+ end
435
+ local parentRawOpts = jobAttributes[3]
436
+ local parentOpts = cjson.decode(parentRawOpts)
437
+ removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
398
438
  end
399
439
  end
400
440
  --[[
@@ -405,12 +445,12 @@ end
405
445
  Function to get max events value or set by default 10000.
406
446
  ]]
407
447
  local function getOrSetMaxEvents(metaKey)
408
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
409
- if not maxEvents then
410
- maxEvents = 10000
411
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
412
- end
413
- return maxEvents
448
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
449
+ if not maxEvents then
450
+ maxEvents = 10000
451
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
452
+ end
453
+ return maxEvents
414
454
  end
415
455
  local function trimEvents(metaKey, eventStreamKey)
416
456
  local maxEvents = getOrSetMaxEvents(metaKey)
@@ -473,7 +513,7 @@ if (#stalling > 0) then
473
513
  rcall("XADD", eventStreamKey, "*", "event",
474
514
  "failed", "jobId", jobId, 'prev', 'active',
475
515
  'failedReason', failedReason)
476
- if rawParentData ~= false then
516
+ if rawParentData then
477
517
  if opts['fpof'] then
478
518
  local parentData = cjson.decode(rawParentData)
479
519
  moveParentFromWaitingChildrenToFailed(