bullmq 5.41.4 → 5.41.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/dist/cjs/classes/job-scheduler.js +40 -15
  2. package/dist/cjs/classes/job-scheduler.js.map +1 -1
  3. package/dist/cjs/classes/job.js +9 -12
  4. package/dist/cjs/classes/job.js.map +1 -1
  5. package/dist/cjs/classes/scripts.js +15 -1
  6. package/dist/cjs/classes/scripts.js.map +1 -1
  7. package/dist/cjs/commands/addDelayedJob-6.lua +5 -2
  8. package/dist/cjs/commands/addJobScheduler-10.lua +130 -0
  9. package/dist/cjs/commands/includes/addDelayedJob.lua +2 -7
  10. package/dist/cjs/commands/includes/getOrSetMaxEvents.lua +6 -6
  11. package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +47 -2
  12. package/dist/cjs/commands/moveStalledJobsToWait-9.lua +1 -1
  13. package/dist/cjs/commands/moveToFinished-14.lua +1 -1
  14. package/dist/cjs/scripts/addDelayedJob-6.js +41 -43
  15. package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
  16. package/dist/cjs/scripts/{addJobScheduler-2.js → addJobScheduler-10.js} +234 -42
  17. package/dist/cjs/scripts/addJobScheduler-10.js.map +1 -0
  18. package/dist/cjs/scripts/addParentJob-4.js +6 -6
  19. package/dist/cjs/scripts/addPrioritizedJob-8.js +6 -6
  20. package/dist/cjs/scripts/addStandardJob-8.js +6 -6
  21. package/dist/cjs/scripts/changeDelay-4.js +6 -6
  22. package/dist/cjs/scripts/index.js +1 -1
  23. package/dist/cjs/scripts/index.js.map +1 -1
  24. package/dist/cjs/scripts/moveJobFromActiveToWait-9.js +6 -6
  25. package/dist/cjs/scripts/moveJobsToWait-8.js +6 -6
  26. package/dist/cjs/scripts/moveStalledJobsToWait-9.js +49 -9
  27. package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
  28. package/dist/cjs/scripts/moveToDelayed-8.js +6 -6
  29. package/dist/cjs/scripts/moveToFinished-14.js +49 -9
  30. package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
  31. package/dist/cjs/scripts/removeJob-3.js +6 -6
  32. package/dist/cjs/scripts/reprocessJob-8.js +6 -6
  33. package/dist/cjs/scripts/retryJob-11.js +6 -6
  34. package/dist/cjs/scripts/updateJobScheduler-7.js +8 -41
  35. package/dist/cjs/scripts/updateJobScheduler-7.js.map +1 -1
  36. package/dist/cjs/scripts/updateProgress-3.js +6 -6
  37. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  38. package/dist/cjs/version.js +1 -1
  39. package/dist/esm/classes/job-scheduler.d.ts +1 -0
  40. package/dist/esm/classes/job-scheduler.js +40 -15
  41. package/dist/esm/classes/job-scheduler.js.map +1 -1
  42. package/dist/esm/classes/job.js +9 -12
  43. package/dist/esm/classes/job.js.map +1 -1
  44. package/dist/esm/classes/scripts.d.ts +2 -2
  45. package/dist/esm/classes/scripts.js +15 -1
  46. package/dist/esm/classes/scripts.js.map +1 -1
  47. package/dist/esm/commands/addDelayedJob-6.lua +5 -2
  48. package/dist/esm/commands/addJobScheduler-10.lua +130 -0
  49. package/dist/esm/commands/includes/addDelayedJob.lua +2 -7
  50. package/dist/esm/commands/includes/getOrSetMaxEvents.lua +6 -6
  51. package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +47 -2
  52. package/dist/esm/commands/moveStalledJobsToWait-9.lua +1 -1
  53. package/dist/esm/commands/moveToFinished-14.lua +1 -1
  54. package/dist/esm/scripts/addDelayedJob-6.js +41 -43
  55. package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
  56. package/dist/esm/scripts/{addJobScheduler-2.js → addJobScheduler-10.js} +234 -42
  57. package/dist/esm/scripts/addJobScheduler-10.js.map +1 -0
  58. package/dist/esm/scripts/addParentJob-4.js +6 -6
  59. package/dist/esm/scripts/addPrioritizedJob-8.js +6 -6
  60. package/dist/esm/scripts/addStandardJob-8.js +6 -6
  61. package/dist/esm/scripts/changeDelay-4.js +6 -6
  62. package/dist/esm/scripts/index.d.ts +1 -1
  63. package/dist/esm/scripts/index.js +1 -1
  64. package/dist/esm/scripts/index.js.map +1 -1
  65. package/dist/esm/scripts/moveJobFromActiveToWait-9.js +6 -6
  66. package/dist/esm/scripts/moveJobsToWait-8.js +6 -6
  67. package/dist/esm/scripts/moveStalledJobsToWait-9.js +49 -9
  68. package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
  69. package/dist/esm/scripts/moveToDelayed-8.js +6 -6
  70. package/dist/esm/scripts/moveToFinished-14.js +49 -9
  71. package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
  72. package/dist/esm/scripts/removeJob-3.js +6 -6
  73. package/dist/esm/scripts/reprocessJob-8.js +6 -6
  74. package/dist/esm/scripts/retryJob-11.js +6 -6
  75. package/dist/esm/scripts/updateJobScheduler-7.js +8 -41
  76. package/dist/esm/scripts/updateJobScheduler-7.js.map +1 -1
  77. package/dist/esm/scripts/updateProgress-3.js +6 -6
  78. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  79. package/dist/esm/version.d.ts +1 -1
  80. package/dist/esm/version.js +1 -1
  81. package/package.json +1 -1
  82. package/dist/cjs/commands/addJobScheduler-2.lua +0 -52
  83. package/dist/cjs/scripts/addJobScheduler-2.js.map +0 -1
  84. package/dist/esm/commands/addJobScheduler-2.lua +0 -52
  85. package/dist/esm/scripts/addJobScheduler-2.js.map +0 -1
  86. /package/dist/esm/scripts/{addJobScheduler-2.d.ts → addJobScheduler-10.d.ts} +0 -0
@@ -1,8 +1,16 @@
1
1
  const content = `--[[
2
2
  Adds a job scheduler, i.e. a job factory that creates jobs based on a given schedule (repeat options).
3
3
  Input:
4
- KEYS[1] 'repeat' key
5
- KEYS[2] 'delayed' key
4
+ KEYS[1] 'repeat' key
5
+ KEYS[2] 'delayed' key
6
+ KEYS[3] 'wait' key
7
+ KEYS[4] 'paused' key
8
+ KEYS[5] 'meta' key
9
+ KEYS[6] 'prioritized' key
10
+ KEYS[7] 'marker' key
11
+ KEYS[8] 'id' key
12
+ KEYS[9] 'events' key
13
+ KEYS[10] 'pc' priority counter
6
14
  ARGV[1] next milliseconds
7
15
  ARGV[2] msgpacked options
8
16
  [1] name
@@ -13,46 +21,88 @@ const content = `--[[
13
21
  ARGV[3] jobs scheduler id
14
22
  ARGV[4] Json stringified template data
15
23
  ARGV[5] mspacked template opts
16
- ARGV[6] prefix key
24
+ ARGV[6] msgpacked delayed opts
25
+ ARGV[7] timestamp
26
+ ARGV[8] prefix key
27
+ ARGV[9] producer key
17
28
  Output:
18
29
  repeatableKey - OK
19
- ]] local rcall = redis.call
30
+ ]]
31
+ local rcall = redis.call
20
32
  local repeatKey = KEYS[1]
21
33
  local delayedKey = KEYS[2]
34
+ local prioritizedKey = KEYS[6]
22
35
  local nextMillis = ARGV[1]
23
36
  local jobSchedulerId = ARGV[3]
24
37
  local templateOpts = cmsgpack.unpack(ARGV[5])
25
- local prefixKey = ARGV[6]
38
+ local prefixKey = ARGV[8]
26
39
  -- Includes
27
40
  --[[
28
- Function to remove job.
41
+ Adds a delayed job to the queue by doing the following:
42
+ - Creates a new job key with the job data.
43
+ - adds to delayed zset.
44
+ - Emits a global event 'delayed' if the job is delayed.
29
45
  ]]
30
46
  -- Includes
31
47
  --[[
32
- Function to remove deduplication key.
48
+ Add delay marker if needed.
33
49
  ]]
34
- local function removeDeduplicationKey(prefixKey, jobKey)
35
- local deduplicationId = rcall("HGET", jobKey, "deid")
36
- if deduplicationId then
37
- local deduplicationKey = prefixKey .. "de:" .. deduplicationId
38
- rcall("DEL", deduplicationKey)
39
- end
40
- end
50
+ -- Includes
41
51
  --[[
42
- Function to remove job keys.
52
+ Function to return the next delayed job timestamp.
43
53
  ]]
44
- local function removeJobKeys(jobKey)
45
- return rcall("DEL", jobKey, jobKey .. ':logs',
46
- jobKey .. ':dependencies', jobKey .. ':processed', jobKey .. ':failed')
54
+ local function getNextDelayedTimestamp(delayedKey)
55
+ local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
56
+ if #result then
57
+ local nextTimestamp = tonumber(result[2])
58
+ if nextTimestamp ~= nil then
59
+ return nextTimestamp / 0x1000
60
+ end
61
+ end
62
+ end
63
+ local function addDelayMarkerIfNeeded(markerKey, delayedKey)
64
+ local nextTimestamp = getNextDelayedTimestamp(delayedKey)
65
+ if nextTimestamp ~= nil then
66
+ -- Replace the score of the marker with the newest known
67
+ -- next timestamp.
68
+ rcall("ZADD", markerKey, nextTimestamp, "1")
69
+ end
47
70
  end
48
71
  --[[
49
- Check if this job has a parent. If so we will just remove it from
50
- the parent child list, but if it is the last child we should move the parent to "wait/paused"
51
- which requires code from "moveToFinished"
72
+ Bake in the job id first 12 bits into the timestamp
73
+ to guarantee correct execution order of delayed jobs
74
+ (up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
75
+ WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
52
76
  ]]
53
- -- Includes
77
+ local function getDelayedScore(delayedKey, timestamp, delay)
78
+ local delayedTimestamp = (delay > 0 and (tonumber(timestamp) + delay)) or tonumber(timestamp)
79
+ local minScore = delayedTimestamp * 0x1000
80
+ local maxScore = (delayedTimestamp + 1 ) * 0x1000 - 1
81
+ local result = rcall("ZREVRANGEBYSCORE", delayedKey, maxScore,
82
+ minScore, "WITHSCORES","LIMIT", 0, 1)
83
+ if #result then
84
+ local currentMaxScore = tonumber(result[2])
85
+ if currentMaxScore ~= nil then
86
+ if currentMaxScore >= maxScore then
87
+ return maxScore, delayedTimestamp
88
+ else
89
+ return currentMaxScore + 1, delayedTimestamp
90
+ end
91
+ end
92
+ end
93
+ return minScore, delayedTimestamp
94
+ end
95
+ local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
96
+ maxEvents, markerKey, delay)
97
+ local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
98
+ rcall("ZADD", delayedKey, score, jobId)
99
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
100
+ "jobId", jobId, "delay", delayedTimestamp)
101
+ -- mark that a delayed job is available
102
+ addDelayMarkerIfNeeded(markerKey, delayedKey)
103
+ end
54
104
  --[[
55
- Function to add job in target list and add marker if needed.
105
+ Function to add job considering priority.
56
106
  ]]
57
107
  -- Includes
58
108
  --[[
@@ -63,19 +113,29 @@ local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
63
113
  rcall("ZADD", markerKey, 0, "0")
64
114
  end
65
115
  end
66
- local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
67
- rcall(pushCmd, targetKey, jobId)
116
+ --[[
117
+ Function to get priority score.
118
+ ]]
119
+ local function getPriorityScore(priority, priorityCounterKey)
120
+ local prioCounter = rcall("INCR", priorityCounterKey)
121
+ return priority * 0x100000000 + prioCounter % 0x100000000
122
+ end
123
+ local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
124
+ isPausedOrMaxed)
125
+ local score = getPriorityScore(priority, priorityCounterKey)
126
+ rcall("ZADD", prioritizedKey, score, jobId)
68
127
  addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
69
128
  end
70
129
  --[[
71
- Functions to destructure job key.
72
- Just a bit of warning, these functions may be a bit slow and affect performance significantly.
130
+ Function to get max events value or set by default 10000.
73
131
  ]]
74
- local getJobIdFromKey = function (jobKey)
75
- return string.match(jobKey, ".*:(.*)")
76
- end
77
- local getJobKeyPrefix = function (jobKey, jobId)
78
- return string.sub(jobKey, 0, #jobKey - #jobId)
132
+ local function getOrSetMaxEvents(metaKey)
133
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
134
+ if not maxEvents then
135
+ maxEvents = 10000
136
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
137
+ end
138
+ return maxEvents
79
139
  end
80
140
  --[[
81
141
  Function to check for the meta.paused key to decide if we are paused or not
@@ -97,6 +157,58 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
97
157
  end
98
158
  return waitKey, false
99
159
  end
160
+ --[[
161
+ Function to check for the meta.paused key to decide if we are paused or not
162
+ (since an empty list and !EXISTS are not really the same).
163
+ ]]
164
+ local function isQueuePaused(queueMetaKey)
165
+ return rcall("HEXISTS", queueMetaKey, "paused") == 1
166
+ end
167
+ --[[
168
+ Function to remove job.
169
+ ]]
170
+ -- Includes
171
+ --[[
172
+ Function to remove deduplication key.
173
+ ]]
174
+ local function removeDeduplicationKey(prefixKey, jobKey)
175
+ local deduplicationId = rcall("HGET", jobKey, "deid")
176
+ if deduplicationId then
177
+ local deduplicationKey = prefixKey .. "de:" .. deduplicationId
178
+ rcall("DEL", deduplicationKey)
179
+ end
180
+ end
181
+ --[[
182
+ Function to remove job keys.
183
+ ]]
184
+ local function removeJobKeys(jobKey)
185
+ return rcall("DEL", jobKey, jobKey .. ':logs',
186
+ jobKey .. ':dependencies', jobKey .. ':processed', jobKey .. ':failed')
187
+ end
188
+ --[[
189
+ Check if this job has a parent. If so we will just remove it from
190
+ the parent child list, but if it is the last child we should move the parent to "wait/paused"
191
+ which requires code from "moveToFinished"
192
+ ]]
193
+ -- Includes
194
+ --[[
195
+ Function to add job in target list and add marker if needed.
196
+ ]]
197
+ -- Includes
198
+ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
199
+ rcall(pushCmd, targetKey, jobId)
200
+ addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
201
+ end
202
+ --[[
203
+ Functions to destructure job key.
204
+ Just a bit of warning, these functions may be a bit slow and affect performance significantly.
205
+ ]]
206
+ local getJobIdFromKey = function (jobKey)
207
+ return string.match(jobKey, ".*:(.*)")
208
+ end
209
+ local getJobKeyPrefix = function (jobKey, jobId)
210
+ return string.sub(jobKey, 0, #jobKey - #jobId)
211
+ end
100
212
  local function moveParentToWait(parentPrefix, parentId, emitEvent)
101
213
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
102
214
  parentPrefix .. "wait", parentPrefix .. "paused")
@@ -177,6 +289,36 @@ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
177
289
  end
178
290
  removeJobKeys(jobKey)
179
291
  end
292
+ --[[
293
+ Function to store a job
294
+ ]]
295
+ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
296
+ parentKey, parentData, repeatJobKey)
297
+ local jsonOpts = cjson.encode(opts)
298
+ local delay = opts['delay'] or 0
299
+ local priority = opts['priority'] or 0
300
+ local debounceId = opts['de'] and opts['de']['id']
301
+ local optionalValues = {}
302
+ if parentKey ~= nil then
303
+ table.insert(optionalValues, "parentKey")
304
+ table.insert(optionalValues, parentKey)
305
+ table.insert(optionalValues, "parent")
306
+ table.insert(optionalValues, parentData)
307
+ end
308
+ if repeatJobKey ~= nil then
309
+ table.insert(optionalValues, "rjk")
310
+ table.insert(optionalValues, repeatJobKey)
311
+ end
312
+ if debounceId then
313
+ table.insert(optionalValues, "deid")
314
+ table.insert(optionalValues, debounceId)
315
+ end
316
+ rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
317
+ "timestamp", timestamp, "delay", delay, "priority", priority,
318
+ unpack(optionalValues))
319
+ rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
320
+ return delay, priority
321
+ end
180
322
  --[[
181
323
  Function to store a job scheduler
182
324
  ]]
@@ -218,23 +360,73 @@ end
218
360
  -- If we are overriding a repeatable job we must delete the delayed job for
219
361
  -- the next iteration.
220
362
  local schedulerKey = repeatKey .. ":" .. jobSchedulerId
363
+ local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
364
+ local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
221
365
  local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
222
366
  if prevMillis ~= false then
223
- local delayedJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
224
- local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
225
- local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
226
- if rcall("ZSCORE", delayedKey, delayedJobId) ~= false and
227
- (rcall("EXISTS", nextDelayedJobKey) ~= 1 or delayedJobId == nextDelayedJobId) then
228
- removeJob(delayedJobId, true, prefixKey, true --[[remove debounce key]] )
229
- rcall("ZREM", delayedKey, delayedJobId)
367
+ local currentJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
368
+ if rcall("EXISTS", nextDelayedJobKey) ~= 1 or currentJobId == nextDelayedJobId then
369
+ if rcall("ZSCORE", delayedKey, currentJobId) ~= false then
370
+ removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
371
+ rcall("ZREM", delayedKey, currentJobId)
372
+ elseif rcall("ZSCORE", prioritizedKey, currentJobId) ~= false then
373
+ removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
374
+ rcall("ZREM", prioritizedKey, currentJobId)
375
+ else
376
+ if isQueuePaused(KEYS[5]) then
377
+ if rcall("LREM", KEYS[4], 1, currentJobId) > 0 then
378
+ removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
379
+ end
380
+ else
381
+ if rcall("LREM", KEYS[3], 1, currentJobId) > 0 then
382
+ removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
383
+ end
384
+ end
385
+ end
230
386
  end
231
387
  end
232
388
  local schedulerOpts = cmsgpack.unpack(ARGV[2])
233
- return storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, ARGV[4], templateOpts)
389
+ storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, ARGV[4], templateOpts)
390
+ if rcall("EXISTS", nextDelayedJobKey) ~= 1 then
391
+ local eventsKey = KEYS[9]
392
+ local metaKey = KEYS[5]
393
+ local maxEvents = getOrSetMaxEvents(metaKey)
394
+ rcall("INCR", KEYS[8])
395
+ local delayedOpts = cmsgpack.unpack(ARGV[6])
396
+ local delay, priority = storeJob(eventsKey, nextDelayedJobKey, nextDelayedJobId, schedulerOpts['name'], ARGV[4],
397
+ delayedOpts, ARGV[7], nil, nil, jobSchedulerId)
398
+ if delay ~= 0 then
399
+ addDelayedJob(nextDelayedJobId, delayedKey, eventsKey,
400
+ ARGV[7], maxEvents, KEYS[7], delay)
401
+ else
402
+ local isPaused = isQueuePaused(KEYS[5])
403
+ -- Standard or priority add
404
+ if priority == 0 then
405
+ if isPaused then
406
+ -- LIFO or FIFO
407
+ local pushCmd = delayedOpts['lifo'] and 'RPUSH' or 'LPUSH'
408
+ rcall(pushCmd, KEYS[4], nextDelayedJobId)
409
+ else
410
+ -- LIFO or FIFO
411
+ local pushCmd = delayedOpts['lifo'] and 'RPUSH' or 'LPUSH'
412
+ rcall(pushCmd, KEYS[3], nextDelayedJobId)
413
+ end
414
+ else
415
+ -- Priority add
416
+ addJobWithPriority(KEYS[7], KEYS[6], priority, nextDelayedJobId, KEYS[10], isPaused)
417
+ end
418
+ -- Emit waiting event
419
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting", "jobId", nextDelayedJobId)
420
+ end
421
+ if ARGV[9] ~= "" then
422
+ rcall("HSET", ARGV[9], "nrjid", nextDelayedJobId)
423
+ end
424
+ return nextDelayedJobId .. "" -- convert to string
425
+ end
234
426
  `;
235
427
  export const addJobScheduler = {
236
428
  name: 'addJobScheduler',
237
429
  content,
238
- keys: 2,
430
+ keys: 10,
239
431
  };
240
- //# sourceMappingURL=addJobScheduler-2.js.map
432
+ //# sourceMappingURL=addJobScheduler-10.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"addJobScheduler-10.js","sourceRoot":"","sources":["../../../src/scripts/addJobScheduler-10.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyaf,CAAC;AACF,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,IAAI,EAAE,iBAAiB;IACvB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
@@ -68,12 +68,12 @@ end
68
68
  Function to get max events value or set by default 10000.
69
69
  ]]
70
70
  local function getOrSetMaxEvents(metaKey)
71
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
72
- if not maxEvents then
73
- maxEvents = 10000
74
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
75
- end
76
- return maxEvents
71
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
72
+ if not maxEvents then
73
+ maxEvents = 10000
74
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
75
+ end
76
+ return maxEvents
77
77
  end
78
78
  --[[
79
79
  Function to handle the case when job is duplicated.
@@ -130,12 +130,12 @@ end
130
130
  Function to get max events value or set by default 10000.
131
131
  ]]
132
132
  local function getOrSetMaxEvents(metaKey)
133
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
134
- if not maxEvents then
135
- maxEvents = 10000
136
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
137
- end
138
- return maxEvents
133
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
134
+ if not maxEvents then
135
+ maxEvents = 10000
136
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
137
+ end
138
+ return maxEvents
139
139
  end
140
140
  --[[
141
141
  Function to handle the case when job is duplicated.
@@ -94,12 +94,12 @@ end
94
94
  Function to get max events value or set by default 10000.
95
95
  ]]
96
96
  local function getOrSetMaxEvents(metaKey)
97
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
98
- if not maxEvents then
99
- maxEvents = 10000
100
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
101
- end
102
- return maxEvents
97
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
98
+ if not maxEvents then
99
+ maxEvents = 10000
100
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
101
+ end
102
+ return maxEvents
103
103
  end
104
104
  --[[
105
105
  Function to check for the meta.paused key to decide if we are paused or not
@@ -70,12 +70,12 @@ end
70
70
  Function to get max events value or set by default 10000.
71
71
  ]]
72
72
  local function getOrSetMaxEvents(metaKey)
73
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
74
- if not maxEvents then
75
- maxEvents = 10000
76
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
77
- end
78
- return maxEvents
73
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
74
+ if not maxEvents then
75
+ maxEvents = 10000
76
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
77
+ end
78
+ return maxEvents
79
79
  end
80
80
  if rcall("EXISTS", ARGV[4]) == 1 then
81
81
  local jobId = ARGV[3]
@@ -1,5 +1,5 @@
1
1
  export * from './addDelayedJob-6';
2
- export * from './addJobScheduler-2';
2
+ export * from './addJobScheduler-10';
3
3
  export * from './addLog-2';
4
4
  export * from './addParentJob-4';
5
5
  export * from './addPrioritizedJob-8';
@@ -1,5 +1,5 @@
1
1
  export * from './addDelayedJob-6';
2
- export * from './addJobScheduler-2';
2
+ export * from './addJobScheduler-10';
3
3
  export * from './addLog-2';
4
4
  export * from './addParentJob-4';
5
5
  export * from './addPrioritizedJob-8';
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/scripts/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,YAAY,CAAC;AAC3B,cAAc,kBAAkB,CAAC;AACjC,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AACnC,cAAc,oBAAoB,CAAC;AACnC,cAAc,WAAW,CAAC;AAC1B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,eAAe,CAAC;AAC9B,cAAc,0BAA0B,CAAC;AACzC,cAAc,qBAAqB,CAAC;AACpC,cAAc,eAAe,CAAC;AAC9B,cAAc,qBAAqB,CAAC;AACpC,cAAc,cAAc,CAAC;AAC7B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,aAAa,CAAC;AAC5B,cAAc,6BAA6B,CAAC;AAC5C,cAAc,oBAAoB,CAAC;AACnC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,mBAAmB,CAAC;AAClC,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,gBAAgB,CAAC;AAC/B,cAAc,cAAc,CAAC;AAC7B,cAAc,WAAW,CAAC;AAC1B,cAAc,aAAa,CAAC;AAC5B,cAAc,iBAAiB,CAAC;AAChC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,eAAe,CAAC;AAC9B,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,kBAAkB,CAAC;AACjC,cAAc,eAAe,CAAC;AAC9B,cAAc,oBAAoB,CAAC;AACnC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,+BAA+B,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/scripts/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,sBAAsB,CAAC;AACrC,cAAc,YAAY,CAAC;AAC3B,cAAc,kBAAkB,CAAC;AACjC,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AACnC,cAAc,oBAAoB,CAAC;AACnC,cAAc,WAAW,CAAC;AAC1B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,eAAe,CAAC;AAC9B,cAAc,0BAA0B,CAAC;AACzC,cAAc,qBAAqB,CAAC;AACpC,cAAc,eAAe,CAAC;AAC9B,cAAc,qBAAqB,CAAC;AACpC,cAAc,cAAc,CAAC;AAC7B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,aAAa,CAAC;AAC5B,cAAc,6BAA6B,CAAC;AAC5C,cAAc,oBAAoB,CAAC;AACnC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,mBAAmB,CAAC;AAClC,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,gBAAgB,CAAC;AAC/B,cAAc,cAAc,CAAC;AAC7B,cAAc,WAAW,CAAC;AAC1B,cAAc,aAAa,CAAC;AAC5B,cAAc,iBAAiB,CAAC;AAChC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,eAAe,CAAC;AAC9B,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,kBAAkB,CAAC;AACjC,cAAc,eAAe,CAAC;AAC9B,cAAc,oBAAoB,CAAC;AACnC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,wBAAwB,CAAC;AACvC,cAAc,oBAAoB,CAAC;AACnC,cAAc,+BAA+B,CAAC"}
@@ -45,12 +45,12 @@ end
45
45
  Function to get max events value or set by default 10000.
46
46
  ]]
47
47
  local function getOrSetMaxEvents(metaKey)
48
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
49
- if not maxEvents then
50
- maxEvents = 10000
51
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
52
- end
53
- return maxEvents
48
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
49
+ if not maxEvents then
50
+ maxEvents = 10000
51
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
52
+ end
53
+ return maxEvents
54
54
  end
55
55
  --[[
56
56
  Function to check for the meta.paused key to decide if we are paused or not
@@ -49,12 +49,12 @@ end
49
49
  Function to get max events value or set by default 10000.
50
50
  ]]
51
51
  local function getOrSetMaxEvents(metaKey)
52
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
53
- if not maxEvents then
54
- maxEvents = 10000
55
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
56
- end
57
- return maxEvents
52
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
53
+ if not maxEvents then
54
+ maxEvents = 10000
55
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
56
+ end
57
+ return maxEvents
58
58
  end
59
59
  --[[
60
60
  Function to check for the meta.paused key to decide if we are paused or not
@@ -358,8 +358,11 @@ local function removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestam
358
358
  end
359
359
  end
360
360
  end
361
- local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
362
- if rcall("ZREM", parentQueueKey .. ":waiting-children", parentId) == 1 then
361
+ local function moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
362
+ local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
363
+ local parentDelayedKey = parentQueueKey .. ":delayed"
364
+ if rcall("ZSCORE", parentWaitingChildrenKey, parentId) ~= false then
365
+ rcall("ZREM", parentWaitingChildrenKey, parentId)
363
366
  local parentQueuePrefix = parentQueueKey .. ":"
364
367
  local parentFailedKey = parentQueueKey .. ":failed"
365
368
  rcall("ZADD", parentFailedKey, timestamp, parentId)
@@ -395,6 +398,43 @@ local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey,
395
398
  local parentRawOpts = jobAttributes[3]
396
399
  local parentOpts = cjson.decode(parentRawOpts)
397
400
  removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
401
+ elseif rcall("ZSCORE", parentDelayedKey, parentId) ~= false then
402
+ rcall("ZREM", parentDelayedKey, parentId)
403
+ local parentQueuePrefix = parentQueueKey .. ":"
404
+ local parentFailedKey = parentQueueKey .. ":failed"
405
+ rcall("ZADD", parentFailedKey, timestamp, parentId)
406
+ local failedReason = "child " .. jobIdKey .. " failed"
407
+ rcall("HMSET", parentKey, "failedReason", failedReason, "finishedOn", timestamp)
408
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason",
409
+ failedReason, "prev", "delayed")
410
+ local jobAttributes = rcall("HMGET", parentKey, "parent", "deid", "opts")
411
+ removeDeduplicationKeyIfNeeded(parentQueueKey .. ":", jobAttributes[2])
412
+ if jobAttributes[1] then
413
+ local parentData = cjson.decode(jobAttributes[1])
414
+ if parentData['fpof'] then
415
+ moveParentFromWaitingChildrenToFailed(
416
+ parentData['queueKey'],
417
+ parentData['queueKey'] .. ':' .. parentData['id'],
418
+ parentData['id'],
419
+ parentKey,
420
+ timestamp
421
+ )
422
+ elseif parentData['idof'] or parentData['rdof'] then
423
+ local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
424
+ local grandParentDependenciesSet = grandParentKey .. ":dependencies"
425
+ if rcall("SREM", grandParentDependenciesSet, parentKey) == 1 then
426
+ moveParentToWaitIfNeeded(parentData['queueKey'], grandParentDependenciesSet,
427
+ grandParentKey, parentData['id'], timestamp)
428
+ if parentData['idof'] then
429
+ local grandParentFailedSet = grandParentKey .. ":failed"
430
+ rcall("HSET", grandParentFailedSet, parentKey, failedReason)
431
+ end
432
+ end
433
+ end
434
+ end
435
+ local parentRawOpts = jobAttributes[3]
436
+ local parentOpts = cjson.decode(parentRawOpts)
437
+ removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
398
438
  end
399
439
  end
400
440
  --[[
@@ -405,12 +445,12 @@ end
405
445
  Function to get max events value or set by default 10000.
406
446
  ]]
407
447
  local function getOrSetMaxEvents(metaKey)
408
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
409
- if not maxEvents then
410
- maxEvents = 10000
411
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
412
- end
413
- return maxEvents
448
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
449
+ if not maxEvents then
450
+ maxEvents = 10000
451
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
452
+ end
453
+ return maxEvents
414
454
  end
415
455
  local function trimEvents(metaKey, eventStreamKey)
416
456
  local maxEvents = getOrSetMaxEvents(metaKey)
@@ -473,7 +513,7 @@ if (#stalling > 0) then
473
513
  rcall("XADD", eventStreamKey, "*", "event",
474
514
  "failed", "jobId", jobId, 'prev', 'active',
475
515
  'failedReason', failedReason)
476
- if rawParentData ~= false then
516
+ if rawParentData then
477
517
  if opts['fpof'] then
478
518
  local parentData = cjson.decode(rawParentData)
479
519
  moveParentFromWaitingChildrenToFailed(
@@ -1 +1 @@
1
- {"version":3,"file":"moveStalledJobsToWait-9.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-9.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yBA6gBS,CAAC;AAC1B,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"moveStalledJobsToWait-9.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-9.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yBAqjBS,CAAC;AAC1B,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -77,12 +77,12 @@ end
77
77
  Function to get max events value or set by default 10000.
78
78
  ]]
79
79
  local function getOrSetMaxEvents(metaKey)
80
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
81
- if not maxEvents then
82
- maxEvents = 10000
83
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
84
- end
85
- return maxEvents
80
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
81
+ if not maxEvents then
82
+ maxEvents = 10000
83
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
84
+ end
85
+ return maxEvents
86
86
  end
87
87
  local function removeLock(jobKey, stalledKey, token, jobId)
88
88
  if token ~= "0" then