bullmq 5.41.4 → 5.41.6

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. package/dist/cjs/classes/job-scheduler.js +40 -15
  2. package/dist/cjs/classes/job-scheduler.js.map +1 -1
  3. package/dist/cjs/classes/job.js +9 -12
  4. package/dist/cjs/classes/job.js.map +1 -1
  5. package/dist/cjs/classes/scripts.js +15 -1
  6. package/dist/cjs/classes/scripts.js.map +1 -1
  7. package/dist/cjs/commands/addDelayedJob-6.lua +5 -2
  8. package/dist/cjs/commands/addJobScheduler-10.lua +130 -0
  9. package/dist/cjs/commands/includes/addDelayedJob.lua +2 -7
  10. package/dist/cjs/commands/includes/getOrSetMaxEvents.lua +6 -6
  11. package/dist/cjs/commands/includes/moveParentFromWaitingChildrenToFailed.lua +47 -2
  12. package/dist/cjs/commands/moveStalledJobsToWait-9.lua +1 -1
  13. package/dist/cjs/commands/moveToFinished-14.lua +1 -1
  14. package/dist/cjs/scripts/addDelayedJob-6.js +41 -43
  15. package/dist/cjs/scripts/addDelayedJob-6.js.map +1 -1
  16. package/dist/cjs/scripts/{addJobScheduler-2.js → addJobScheduler-10.js} +234 -42
  17. package/dist/cjs/scripts/addJobScheduler-10.js.map +1 -0
  18. package/dist/cjs/scripts/addParentJob-4.js +6 -6
  19. package/dist/cjs/scripts/addPrioritizedJob-8.js +6 -6
  20. package/dist/cjs/scripts/addStandardJob-8.js +6 -6
  21. package/dist/cjs/scripts/changeDelay-4.js +6 -6
  22. package/dist/cjs/scripts/index.js +1 -1
  23. package/dist/cjs/scripts/index.js.map +1 -1
  24. package/dist/cjs/scripts/moveJobFromActiveToWait-9.js +6 -6
  25. package/dist/cjs/scripts/moveJobsToWait-8.js +6 -6
  26. package/dist/cjs/scripts/moveStalledJobsToWait-9.js +49 -9
  27. package/dist/cjs/scripts/moveStalledJobsToWait-9.js.map +1 -1
  28. package/dist/cjs/scripts/moveToDelayed-8.js +6 -6
  29. package/dist/cjs/scripts/moveToFinished-14.js +49 -9
  30. package/dist/cjs/scripts/moveToFinished-14.js.map +1 -1
  31. package/dist/cjs/scripts/removeJob-3.js +6 -6
  32. package/dist/cjs/scripts/reprocessJob-8.js +6 -6
  33. package/dist/cjs/scripts/retryJob-11.js +6 -6
  34. package/dist/cjs/scripts/updateJobScheduler-7.js +8 -41
  35. package/dist/cjs/scripts/updateJobScheduler-7.js.map +1 -1
  36. package/dist/cjs/scripts/updateProgress-3.js +6 -6
  37. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  38. package/dist/cjs/version.js +1 -1
  39. package/dist/esm/classes/job-scheduler.d.ts +1 -0
  40. package/dist/esm/classes/job-scheduler.js +40 -15
  41. package/dist/esm/classes/job-scheduler.js.map +1 -1
  42. package/dist/esm/classes/job.js +9 -12
  43. package/dist/esm/classes/job.js.map +1 -1
  44. package/dist/esm/classes/scripts.d.ts +2 -2
  45. package/dist/esm/classes/scripts.js +15 -1
  46. package/dist/esm/classes/scripts.js.map +1 -1
  47. package/dist/esm/commands/addDelayedJob-6.lua +5 -2
  48. package/dist/esm/commands/addJobScheduler-10.lua +130 -0
  49. package/dist/esm/commands/includes/addDelayedJob.lua +2 -7
  50. package/dist/esm/commands/includes/getOrSetMaxEvents.lua +6 -6
  51. package/dist/esm/commands/includes/moveParentFromWaitingChildrenToFailed.lua +47 -2
  52. package/dist/esm/commands/moveStalledJobsToWait-9.lua +1 -1
  53. package/dist/esm/commands/moveToFinished-14.lua +1 -1
  54. package/dist/esm/scripts/addDelayedJob-6.js +41 -43
  55. package/dist/esm/scripts/addDelayedJob-6.js.map +1 -1
  56. package/dist/esm/scripts/{addJobScheduler-2.js → addJobScheduler-10.js} +234 -42
  57. package/dist/esm/scripts/addJobScheduler-10.js.map +1 -0
  58. package/dist/esm/scripts/addParentJob-4.js +6 -6
  59. package/dist/esm/scripts/addPrioritizedJob-8.js +6 -6
  60. package/dist/esm/scripts/addStandardJob-8.js +6 -6
  61. package/dist/esm/scripts/changeDelay-4.js +6 -6
  62. package/dist/esm/scripts/index.d.ts +1 -1
  63. package/dist/esm/scripts/index.js +1 -1
  64. package/dist/esm/scripts/index.js.map +1 -1
  65. package/dist/esm/scripts/moveJobFromActiveToWait-9.js +6 -6
  66. package/dist/esm/scripts/moveJobsToWait-8.js +6 -6
  67. package/dist/esm/scripts/moveStalledJobsToWait-9.js +49 -9
  68. package/dist/esm/scripts/moveStalledJobsToWait-9.js.map +1 -1
  69. package/dist/esm/scripts/moveToDelayed-8.js +6 -6
  70. package/dist/esm/scripts/moveToFinished-14.js +49 -9
  71. package/dist/esm/scripts/moveToFinished-14.js.map +1 -1
  72. package/dist/esm/scripts/removeJob-3.js +6 -6
  73. package/dist/esm/scripts/reprocessJob-8.js +6 -6
  74. package/dist/esm/scripts/retryJob-11.js +6 -6
  75. package/dist/esm/scripts/updateJobScheduler-7.js +8 -41
  76. package/dist/esm/scripts/updateJobScheduler-7.js.map +1 -1
  77. package/dist/esm/scripts/updateProgress-3.js +6 -6
  78. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  79. package/dist/esm/version.d.ts +1 -1
  80. package/dist/esm/version.js +1 -1
  81. package/package.json +1 -1
  82. package/dist/cjs/commands/addJobScheduler-2.lua +0 -52
  83. package/dist/cjs/scripts/addJobScheduler-2.js.map +0 -1
  84. package/dist/esm/commands/addJobScheduler-2.lua +0 -52
  85. package/dist/esm/scripts/addJobScheduler-2.js.map +0 -1
  86. /package/dist/esm/scripts/{addJobScheduler-2.d.ts → addJobScheduler-10.d.ts} +0 -0
@@ -4,8 +4,16 @@ exports.addJobScheduler = void 0;
4
4
  const content = `--[[
5
5
  Adds a job scheduler, i.e. a job factory that creates jobs based on a given schedule (repeat options).
6
6
  Input:
7
- KEYS[1] 'repeat' key
8
- KEYS[2] 'delayed' key
7
+ KEYS[1] 'repeat' key
8
+ KEYS[2] 'delayed' key
9
+ KEYS[3] 'wait' key
10
+ KEYS[4] 'paused' key
11
+ KEYS[5] 'meta' key
12
+ KEYS[6] 'prioritized' key
13
+ KEYS[7] 'marker' key
14
+ KEYS[8] 'id' key
15
+ KEYS[9] 'events' key
16
+ KEYS[10] 'pc' priority counter
9
17
  ARGV[1] next milliseconds
10
18
  ARGV[2] msgpacked options
11
19
  [1] name
@@ -16,46 +24,88 @@ const content = `--[[
16
24
  ARGV[3] jobs scheduler id
17
25
  ARGV[4] Json stringified template data
18
26
  ARGV[5] mspacked template opts
19
- ARGV[6] prefix key
27
+ ARGV[6] msgpacked delayed opts
28
+ ARGV[7] timestamp
29
+ ARGV[8] prefix key
30
+ ARGV[9] producer key
20
31
  Output:
21
32
  repeatableKey - OK
22
- ]] local rcall = redis.call
33
+ ]]
34
+ local rcall = redis.call
23
35
  local repeatKey = KEYS[1]
24
36
  local delayedKey = KEYS[2]
37
+ local prioritizedKey = KEYS[6]
25
38
  local nextMillis = ARGV[1]
26
39
  local jobSchedulerId = ARGV[3]
27
40
  local templateOpts = cmsgpack.unpack(ARGV[5])
28
- local prefixKey = ARGV[6]
41
+ local prefixKey = ARGV[8]
29
42
  -- Includes
30
43
  --[[
31
- Function to remove job.
44
+ Adds a delayed job to the queue by doing the following:
45
+ - Creates a new job key with the job data.
46
+ - adds to delayed zset.
47
+ - Emits a global event 'delayed' if the job is delayed.
32
48
  ]]
33
49
  -- Includes
34
50
  --[[
35
- Function to remove deduplication key.
51
+ Add delay marker if needed.
36
52
  ]]
37
- local function removeDeduplicationKey(prefixKey, jobKey)
38
- local deduplicationId = rcall("HGET", jobKey, "deid")
39
- if deduplicationId then
40
- local deduplicationKey = prefixKey .. "de:" .. deduplicationId
41
- rcall("DEL", deduplicationKey)
42
- end
43
- end
53
+ -- Includes
44
54
  --[[
45
- Function to remove job keys.
55
+ Function to return the next delayed job timestamp.
46
56
  ]]
47
- local function removeJobKeys(jobKey)
48
- return rcall("DEL", jobKey, jobKey .. ':logs',
49
- jobKey .. ':dependencies', jobKey .. ':processed', jobKey .. ':failed')
57
+ local function getNextDelayedTimestamp(delayedKey)
58
+ local result = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")
59
+ if #result then
60
+ local nextTimestamp = tonumber(result[2])
61
+ if nextTimestamp ~= nil then
62
+ return nextTimestamp / 0x1000
63
+ end
64
+ end
65
+ end
66
+ local function addDelayMarkerIfNeeded(markerKey, delayedKey)
67
+ local nextTimestamp = getNextDelayedTimestamp(delayedKey)
68
+ if nextTimestamp ~= nil then
69
+ -- Replace the score of the marker with the newest known
70
+ -- next timestamp.
71
+ rcall("ZADD", markerKey, nextTimestamp, "1")
72
+ end
50
73
  end
51
74
  --[[
52
- Check if this job has a parent. If so we will just remove it from
53
- the parent child list, but if it is the last child we should move the parent to "wait/paused"
54
- which requires code from "moveToFinished"
75
+ Bake in the job id first 12 bits into the timestamp
76
+ to guarantee correct execution order of delayed jobs
77
+ (up to 4096 jobs per given timestamp or 4096 jobs apart per timestamp)
78
+ WARNING: Jobs that are so far apart that they wrap around will cause FIFO to fail
55
79
  ]]
56
- -- Includes
80
+ local function getDelayedScore(delayedKey, timestamp, delay)
81
+ local delayedTimestamp = (delay > 0 and (tonumber(timestamp) + delay)) or tonumber(timestamp)
82
+ local minScore = delayedTimestamp * 0x1000
83
+ local maxScore = (delayedTimestamp + 1 ) * 0x1000 - 1
84
+ local result = rcall("ZREVRANGEBYSCORE", delayedKey, maxScore,
85
+ minScore, "WITHSCORES","LIMIT", 0, 1)
86
+ if #result then
87
+ local currentMaxScore = tonumber(result[2])
88
+ if currentMaxScore ~= nil then
89
+ if currentMaxScore >= maxScore then
90
+ return maxScore, delayedTimestamp
91
+ else
92
+ return currentMaxScore + 1, delayedTimestamp
93
+ end
94
+ end
95
+ end
96
+ return minScore, delayedTimestamp
97
+ end
98
+ local function addDelayedJob(jobId, delayedKey, eventsKey, timestamp,
99
+ maxEvents, markerKey, delay)
100
+ local score, delayedTimestamp = getDelayedScore(delayedKey, timestamp, tonumber(delay))
101
+ rcall("ZADD", delayedKey, score, jobId)
102
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "delayed",
103
+ "jobId", jobId, "delay", delayedTimestamp)
104
+ -- mark that a delayed job is available
105
+ addDelayMarkerIfNeeded(markerKey, delayedKey)
106
+ end
57
107
  --[[
58
- Function to add job in target list and add marker if needed.
108
+ Function to add job considering priority.
59
109
  ]]
60
110
  -- Includes
61
111
  --[[
@@ -66,19 +116,29 @@ local function addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
66
116
  rcall("ZADD", markerKey, 0, "0")
67
117
  end
68
118
  end
69
- local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
70
- rcall(pushCmd, targetKey, jobId)
119
+ --[[
120
+ Function to get priority score.
121
+ ]]
122
+ local function getPriorityScore(priority, priorityCounterKey)
123
+ local prioCounter = rcall("INCR", priorityCounterKey)
124
+ return priority * 0x100000000 + prioCounter % 0x100000000
125
+ end
126
+ local function addJobWithPriority(markerKey, prioritizedKey, priority, jobId, priorityCounterKey,
127
+ isPausedOrMaxed)
128
+ local score = getPriorityScore(priority, priorityCounterKey)
129
+ rcall("ZADD", prioritizedKey, score, jobId)
71
130
  addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
72
131
  end
73
132
  --[[
74
- Functions to destructure job key.
75
- Just a bit of warning, these functions may be a bit slow and affect performance significantly.
133
+ Function to get max events value or set by default 10000.
76
134
  ]]
77
- local getJobIdFromKey = function (jobKey)
78
- return string.match(jobKey, ".*:(.*)")
79
- end
80
- local getJobKeyPrefix = function (jobKey, jobId)
81
- return string.sub(jobKey, 0, #jobKey - #jobId)
135
+ local function getOrSetMaxEvents(metaKey)
136
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
137
+ if not maxEvents then
138
+ maxEvents = 10000
139
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
140
+ end
141
+ return maxEvents
82
142
  end
83
143
  --[[
84
144
  Function to check for the meta.paused key to decide if we are paused or not
@@ -100,6 +160,58 @@ local function getTargetQueueList(queueMetaKey, activeKey, waitKey, pausedKey)
100
160
  end
101
161
  return waitKey, false
102
162
  end
163
+ --[[
164
+ Function to check for the meta.paused key to decide if we are paused or not
165
+ (since an empty list and !EXISTS are not really the same).
166
+ ]]
167
+ local function isQueuePaused(queueMetaKey)
168
+ return rcall("HEXISTS", queueMetaKey, "paused") == 1
169
+ end
170
+ --[[
171
+ Function to remove job.
172
+ ]]
173
+ -- Includes
174
+ --[[
175
+ Function to remove deduplication key.
176
+ ]]
177
+ local function removeDeduplicationKey(prefixKey, jobKey)
178
+ local deduplicationId = rcall("HGET", jobKey, "deid")
179
+ if deduplicationId then
180
+ local deduplicationKey = prefixKey .. "de:" .. deduplicationId
181
+ rcall("DEL", deduplicationKey)
182
+ end
183
+ end
184
+ --[[
185
+ Function to remove job keys.
186
+ ]]
187
+ local function removeJobKeys(jobKey)
188
+ return rcall("DEL", jobKey, jobKey .. ':logs',
189
+ jobKey .. ':dependencies', jobKey .. ':processed', jobKey .. ':failed')
190
+ end
191
+ --[[
192
+ Check if this job has a parent. If so we will just remove it from
193
+ the parent child list, but if it is the last child we should move the parent to "wait/paused"
194
+ which requires code from "moveToFinished"
195
+ ]]
196
+ -- Includes
197
+ --[[
198
+ Function to add job in target list and add marker if needed.
199
+ ]]
200
+ -- Includes
201
+ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed, jobId)
202
+ rcall(pushCmd, targetKey, jobId)
203
+ addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
204
+ end
205
+ --[[
206
+ Functions to destructure job key.
207
+ Just a bit of warning, these functions may be a bit slow and affect performance significantly.
208
+ ]]
209
+ local getJobIdFromKey = function (jobKey)
210
+ return string.match(jobKey, ".*:(.*)")
211
+ end
212
+ local getJobKeyPrefix = function (jobKey, jobId)
213
+ return string.sub(jobKey, 0, #jobKey - #jobId)
214
+ end
103
215
  local function moveParentToWait(parentPrefix, parentId, emitEvent)
104
216
  local parentTarget, isPausedOrMaxed = getTargetQueueList(parentPrefix .. "meta", parentPrefix .. "active",
105
217
  parentPrefix .. "wait", parentPrefix .. "paused")
@@ -180,6 +292,36 @@ local function removeJob(jobId, hard, baseKey, shouldRemoveDeduplicationKey)
180
292
  end
181
293
  removeJobKeys(jobKey)
182
294
  end
295
+ --[[
296
+ Function to store a job
297
+ ]]
298
+ local function storeJob(eventsKey, jobIdKey, jobId, name, data, opts, timestamp,
299
+ parentKey, parentData, repeatJobKey)
300
+ local jsonOpts = cjson.encode(opts)
301
+ local delay = opts['delay'] or 0
302
+ local priority = opts['priority'] or 0
303
+ local debounceId = opts['de'] and opts['de']['id']
304
+ local optionalValues = {}
305
+ if parentKey ~= nil then
306
+ table.insert(optionalValues, "parentKey")
307
+ table.insert(optionalValues, parentKey)
308
+ table.insert(optionalValues, "parent")
309
+ table.insert(optionalValues, parentData)
310
+ end
311
+ if repeatJobKey ~= nil then
312
+ table.insert(optionalValues, "rjk")
313
+ table.insert(optionalValues, repeatJobKey)
314
+ end
315
+ if debounceId then
316
+ table.insert(optionalValues, "deid")
317
+ table.insert(optionalValues, debounceId)
318
+ end
319
+ rcall("HMSET", jobIdKey, "name", name, "data", data, "opts", jsonOpts,
320
+ "timestamp", timestamp, "delay", delay, "priority", priority,
321
+ unpack(optionalValues))
322
+ rcall("XADD", eventsKey, "*", "event", "added", "jobId", jobId, "name", name)
323
+ return delay, priority
324
+ end
183
325
  --[[
184
326
  Function to store a job scheduler
185
327
  ]]
@@ -221,23 +363,73 @@ end
221
363
  -- If we are overriding a repeatable job we must delete the delayed job for
222
364
  -- the next iteration.
223
365
  local schedulerKey = repeatKey .. ":" .. jobSchedulerId
366
+ local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
367
+ local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
224
368
  local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
225
369
  if prevMillis ~= false then
226
- local delayedJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
227
- local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
228
- local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
229
- if rcall("ZSCORE", delayedKey, delayedJobId) ~= false and
230
- (rcall("EXISTS", nextDelayedJobKey) ~= 1 or delayedJobId == nextDelayedJobId) then
231
- removeJob(delayedJobId, true, prefixKey, true --[[remove debounce key]] )
232
- rcall("ZREM", delayedKey, delayedJobId)
370
+ local currentJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
371
+ if rcall("EXISTS", nextDelayedJobKey) ~= 1 or currentJobId == nextDelayedJobId then
372
+ if rcall("ZSCORE", delayedKey, currentJobId) ~= false then
373
+ removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
374
+ rcall("ZREM", delayedKey, currentJobId)
375
+ elseif rcall("ZSCORE", prioritizedKey, currentJobId) ~= false then
376
+ removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
377
+ rcall("ZREM", prioritizedKey, currentJobId)
378
+ else
379
+ if isQueuePaused(KEYS[5]) then
380
+ if rcall("LREM", KEYS[4], 1, currentJobId) > 0 then
381
+ removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
382
+ end
383
+ else
384
+ if rcall("LREM", KEYS[3], 1, currentJobId) > 0 then
385
+ removeJob(currentJobId, true, prefixKey, true --[[remove debounce key]] )
386
+ end
387
+ end
388
+ end
233
389
  end
234
390
  end
235
391
  local schedulerOpts = cmsgpack.unpack(ARGV[2])
236
- return storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, ARGV[4], templateOpts)
392
+ storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, ARGV[4], templateOpts)
393
+ if rcall("EXISTS", nextDelayedJobKey) ~= 1 then
394
+ local eventsKey = KEYS[9]
395
+ local metaKey = KEYS[5]
396
+ local maxEvents = getOrSetMaxEvents(metaKey)
397
+ rcall("INCR", KEYS[8])
398
+ local delayedOpts = cmsgpack.unpack(ARGV[6])
399
+ local delay, priority = storeJob(eventsKey, nextDelayedJobKey, nextDelayedJobId, schedulerOpts['name'], ARGV[4],
400
+ delayedOpts, ARGV[7], nil, nil, jobSchedulerId)
401
+ if delay ~= 0 then
402
+ addDelayedJob(nextDelayedJobId, delayedKey, eventsKey,
403
+ ARGV[7], maxEvents, KEYS[7], delay)
404
+ else
405
+ local isPaused = isQueuePaused(KEYS[5])
406
+ -- Standard or priority add
407
+ if priority == 0 then
408
+ if isPaused then
409
+ -- LIFO or FIFO
410
+ local pushCmd = delayedOpts['lifo'] and 'RPUSH' or 'LPUSH'
411
+ rcall(pushCmd, KEYS[4], nextDelayedJobId)
412
+ else
413
+ -- LIFO or FIFO
414
+ local pushCmd = delayedOpts['lifo'] and 'RPUSH' or 'LPUSH'
415
+ rcall(pushCmd, KEYS[3], nextDelayedJobId)
416
+ end
417
+ else
418
+ -- Priority add
419
+ addJobWithPriority(KEYS[7], KEYS[6], priority, nextDelayedJobId, KEYS[10], isPaused)
420
+ end
421
+ -- Emit waiting event
422
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "waiting", "jobId", nextDelayedJobId)
423
+ end
424
+ if ARGV[9] ~= "" then
425
+ rcall("HSET", ARGV[9], "nrjid", nextDelayedJobId)
426
+ end
427
+ return nextDelayedJobId .. "" -- convert to string
428
+ end
237
429
  `;
238
430
  exports.addJobScheduler = {
239
431
  name: 'addJobScheduler',
240
432
  content,
241
- keys: 2,
433
+ keys: 10,
242
434
  };
243
- //# sourceMappingURL=addJobScheduler-2.js.map
435
+ //# sourceMappingURL=addJobScheduler-10.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"addJobScheduler-10.js","sourceRoot":"","sources":["../../../src/scripts/addJobScheduler-10.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyaf,CAAC;AACW,QAAA,eAAe,GAAG;IAC7B,IAAI,EAAE,iBAAiB;IACvB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
@@ -71,12 +71,12 @@ end
71
71
  Function to get max events value or set by default 10000.
72
72
  ]]
73
73
  local function getOrSetMaxEvents(metaKey)
74
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
75
- if not maxEvents then
76
- maxEvents = 10000
77
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
78
- end
79
- return maxEvents
74
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
75
+ if not maxEvents then
76
+ maxEvents = 10000
77
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
78
+ end
79
+ return maxEvents
80
80
  end
81
81
  --[[
82
82
  Function to handle the case when job is duplicated.
@@ -133,12 +133,12 @@ end
133
133
  Function to get max events value or set by default 10000.
134
134
  ]]
135
135
  local function getOrSetMaxEvents(metaKey)
136
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
137
- if not maxEvents then
138
- maxEvents = 10000
139
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
140
- end
141
- return maxEvents
136
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
137
+ if not maxEvents then
138
+ maxEvents = 10000
139
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
140
+ end
141
+ return maxEvents
142
142
  end
143
143
  --[[
144
144
  Function to handle the case when job is duplicated.
@@ -97,12 +97,12 @@ end
97
97
  Function to get max events value or set by default 10000.
98
98
  ]]
99
99
  local function getOrSetMaxEvents(metaKey)
100
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
101
- if not maxEvents then
102
- maxEvents = 10000
103
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
104
- end
105
- return maxEvents
100
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
101
+ if not maxEvents then
102
+ maxEvents = 10000
103
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
104
+ end
105
+ return maxEvents
106
106
  end
107
107
  --[[
108
108
  Function to check for the meta.paused key to decide if we are paused or not
@@ -73,12 +73,12 @@ end
73
73
  Function to get max events value or set by default 10000.
74
74
  ]]
75
75
  local function getOrSetMaxEvents(metaKey)
76
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
77
- if not maxEvents then
78
- maxEvents = 10000
79
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
80
- end
81
- return maxEvents
76
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
77
+ if not maxEvents then
78
+ maxEvents = 10000
79
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
80
+ end
81
+ return maxEvents
82
82
  end
83
83
  if rcall("EXISTS", ARGV[4]) == 1 then
84
84
  local jobId = ARGV[3]
@@ -2,7 +2,7 @@
2
2
  Object.defineProperty(exports, "__esModule", { value: true });
3
3
  const tslib_1 = require("tslib");
4
4
  tslib_1.__exportStar(require("./addDelayedJob-6"), exports);
5
- tslib_1.__exportStar(require("./addJobScheduler-2"), exports);
5
+ tslib_1.__exportStar(require("./addJobScheduler-10"), exports);
6
6
  tslib_1.__exportStar(require("./addLog-2"), exports);
7
7
  tslib_1.__exportStar(require("./addParentJob-4"), exports);
8
8
  tslib_1.__exportStar(require("./addPrioritizedJob-8"), exports);
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/scripts/index.ts"],"names":[],"mappings":";;;AAAA,4DAAkC;AAClC,8DAAoC;AACpC,qDAA2B;AAC3B,2DAAiC;AACjC,gEAAsC;AACtC,+DAAqC;AACrC,6DAAmC;AACnC,0DAAgC;AAChC,6DAAmC;AACnC,6DAAmC;AACnC,oDAA0B;AAC1B,yDAA+B;AAC/B,0DAAgC;AAChC,wDAA8B;AAC9B,mEAAyC;AACzC,8DAAoC;AACpC,wDAA8B;AAC9B,8DAAoC;AACpC,uDAA6B;AAC7B,yDAA+B;AAC/B,yDAA+B;AAC/B,0DAAgC;AAChC,sDAA4B;AAC5B,sEAA4C;AAC5C,6DAAmC;AACnC,oEAA0C;AAC1C,4DAAkC;AAClC,4DAAkC;AAClC,8DAAoC;AACpC,oEAA0C;AAC1C,yDAA+B;AAC/B,uDAA6B;AAC7B,oDAA0B;AAC1B,sDAA4B;AAC5B,0DAAgC;AAChC,oEAA0C;AAC1C,wDAA8B;AAC9B,iEAAuC;AACvC,+DAAqC;AACrC,2DAAiC;AACjC,wDAA8B;AAC9B,6DAAmC;AACnC,yDAA+B;AAC/B,iEAAuC;AACvC,6DAAmC;AACnC,wEAA8C"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/scripts/index.ts"],"names":[],"mappings":";;;AAAA,4DAAkC;AAClC,+DAAqC;AACrC,qDAA2B;AAC3B,2DAAiC;AACjC,gEAAsC;AACtC,+DAAqC;AACrC,6DAAmC;AACnC,0DAAgC;AAChC,6DAAmC;AACnC,6DAAmC;AACnC,oDAA0B;AAC1B,yDAA+B;AAC/B,0DAAgC;AAChC,wDAA8B;AAC9B,mEAAyC;AACzC,8DAAoC;AACpC,wDAA8B;AAC9B,8DAAoC;AACpC,uDAA6B;AAC7B,yDAA+B;AAC/B,yDAA+B;AAC/B,0DAAgC;AAChC,sDAA4B;AAC5B,sEAA4C;AAC5C,6DAAmC;AACnC,oEAA0C;AAC1C,4DAAkC;AAClC,4DAAkC;AAClC,8DAAoC;AACpC,oEAA0C;AAC1C,yDAA+B;AAC/B,uDAA6B;AAC7B,oDAA0B;AAC1B,sDAA4B;AAC5B,0DAAgC;AAChC,oEAA0C;AAC1C,wDAA8B;AAC9B,iEAAuC;AACvC,+DAAqC;AACrC,2DAAiC;AACjC,wDAA8B;AAC9B,6DAAmC;AACnC,yDAA+B;AAC/B,iEAAuC;AACvC,6DAAmC;AACnC,wEAA8C"}
@@ -48,12 +48,12 @@ end
48
48
  Function to get max events value or set by default 10000.
49
49
  ]]
50
50
  local function getOrSetMaxEvents(metaKey)
51
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
52
- if not maxEvents then
53
- maxEvents = 10000
54
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
55
- end
56
- return maxEvents
51
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
52
+ if not maxEvents then
53
+ maxEvents = 10000
54
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
55
+ end
56
+ return maxEvents
57
57
  end
58
58
  --[[
59
59
  Function to check for the meta.paused key to decide if we are paused or not
@@ -52,12 +52,12 @@ end
52
52
  Function to get max events value or set by default 10000.
53
53
  ]]
54
54
  local function getOrSetMaxEvents(metaKey)
55
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
56
- if not maxEvents then
57
- maxEvents = 10000
58
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
59
- end
60
- return maxEvents
55
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
56
+ if not maxEvents then
57
+ maxEvents = 10000
58
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
59
+ end
60
+ return maxEvents
61
61
  end
62
62
  --[[
63
63
  Function to check for the meta.paused key to decide if we are paused or not
@@ -361,8 +361,11 @@ local function removeJobsOnFail(queueKeyPrefix, failedKey, jobId, opts, timestam
361
361
  end
362
362
  end
363
363
  end
364
- local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
365
- if rcall("ZREM", parentQueueKey .. ":waiting-children", parentId) == 1 then
364
+ local function moveParentFromWaitingChildrenToFailed(parentQueueKey, parentKey, parentId, jobIdKey, timestamp)
365
+ local parentWaitingChildrenKey = parentQueueKey .. ":waiting-children"
366
+ local parentDelayedKey = parentQueueKey .. ":delayed"
367
+ if rcall("ZSCORE", parentWaitingChildrenKey, parentId) ~= false then
368
+ rcall("ZREM", parentWaitingChildrenKey, parentId)
366
369
  local parentQueuePrefix = parentQueueKey .. ":"
367
370
  local parentFailedKey = parentQueueKey .. ":failed"
368
371
  rcall("ZADD", parentFailedKey, timestamp, parentId)
@@ -398,6 +401,43 @@ local function moveParentFromWaitingChildrenToFailed( parentQueueKey, parentKey,
398
401
  local parentRawOpts = jobAttributes[3]
399
402
  local parentOpts = cjson.decode(parentRawOpts)
400
403
  removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
404
+ elseif rcall("ZSCORE", parentDelayedKey, parentId) ~= false then
405
+ rcall("ZREM", parentDelayedKey, parentId)
406
+ local parentQueuePrefix = parentQueueKey .. ":"
407
+ local parentFailedKey = parentQueueKey .. ":failed"
408
+ rcall("ZADD", parentFailedKey, timestamp, parentId)
409
+ local failedReason = "child " .. jobIdKey .. " failed"
410
+ rcall("HMSET", parentKey, "failedReason", failedReason, "finishedOn", timestamp)
411
+ rcall("XADD", parentQueueKey .. ":events", "*", "event", "failed", "jobId", parentId, "failedReason",
412
+ failedReason, "prev", "delayed")
413
+ local jobAttributes = rcall("HMGET", parentKey, "parent", "deid", "opts")
414
+ removeDeduplicationKeyIfNeeded(parentQueueKey .. ":", jobAttributes[2])
415
+ if jobAttributes[1] then
416
+ local parentData = cjson.decode(jobAttributes[1])
417
+ if parentData['fpof'] then
418
+ moveParentFromWaitingChildrenToFailed(
419
+ parentData['queueKey'],
420
+ parentData['queueKey'] .. ':' .. parentData['id'],
421
+ parentData['id'],
422
+ parentKey,
423
+ timestamp
424
+ )
425
+ elseif parentData['idof'] or parentData['rdof'] then
426
+ local grandParentKey = parentData['queueKey'] .. ':' .. parentData['id']
427
+ local grandParentDependenciesSet = grandParentKey .. ":dependencies"
428
+ if rcall("SREM", grandParentDependenciesSet, parentKey) == 1 then
429
+ moveParentToWaitIfNeeded(parentData['queueKey'], grandParentDependenciesSet,
430
+ grandParentKey, parentData['id'], timestamp)
431
+ if parentData['idof'] then
432
+ local grandParentFailedSet = grandParentKey .. ":failed"
433
+ rcall("HSET", grandParentFailedSet, parentKey, failedReason)
434
+ end
435
+ end
436
+ end
437
+ end
438
+ local parentRawOpts = jobAttributes[3]
439
+ local parentOpts = cjson.decode(parentRawOpts)
440
+ removeJobsOnFail(parentQueuePrefix, parentFailedKey, parentId, parentOpts, timestamp)
401
441
  end
402
442
  end
403
443
  --[[
@@ -408,12 +448,12 @@ end
408
448
  Function to get max events value or set by default 10000.
409
449
  ]]
410
450
  local function getOrSetMaxEvents(metaKey)
411
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
412
- if not maxEvents then
413
- maxEvents = 10000
414
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
415
- end
416
- return maxEvents
451
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
452
+ if not maxEvents then
453
+ maxEvents = 10000
454
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
455
+ end
456
+ return maxEvents
417
457
  end
418
458
  local function trimEvents(metaKey, eventStreamKey)
419
459
  local maxEvents = getOrSetMaxEvents(metaKey)
@@ -476,7 +516,7 @@ if (#stalling > 0) then
476
516
  rcall("XADD", eventStreamKey, "*", "event",
477
517
  "failed", "jobId", jobId, 'prev', 'active',
478
518
  'failedReason', failedReason)
479
- if rawParentData ~= false then
519
+ if rawParentData then
480
520
  if opts['fpof'] then
481
521
  local parentData = cjson.decode(rawParentData)
482
522
  moveParentFromWaitingChildrenToFailed(
@@ -1 +1 @@
1
- {"version":3,"file":"moveStalledJobsToWait-9.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yBA6gBS,CAAC;AACb,QAAA,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"moveStalledJobsToWait-9.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;yBAqjBS,CAAC;AACb,QAAA,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -80,12 +80,12 @@ end
80
80
  Function to get max events value or set by default 10000.
81
81
  ]]
82
82
  local function getOrSetMaxEvents(metaKey)
83
- local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
84
- if not maxEvents then
85
- maxEvents = 10000
86
- rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
87
- end
88
- return maxEvents
83
+ local maxEvents = rcall("HGET", metaKey, "opts.maxLenEvents")
84
+ if not maxEvents then
85
+ maxEvents = 10000
86
+ rcall("HSET", metaKey, "opts.maxLenEvents", maxEvents)
87
+ end
88
+ return maxEvents
89
89
  end
90
90
  local function removeLock(jobKey, stalledKey, token, jobId)
91
91
  if token ~= "0" then