bullmq 5.58.6 → 5.58.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. package/dist/cjs/classes/job-scheduler.js +22 -24
  2. package/dist/cjs/classes/job-scheduler.js.map +1 -1
  3. package/dist/cjs/classes/job.js +19 -7
  4. package/dist/cjs/classes/job.js.map +1 -1
  5. package/dist/cjs/classes/queue.js +1 -1
  6. package/dist/cjs/classes/queue.js.map +1 -1
  7. package/dist/cjs/classes/scripts.js +37 -11
  8. package/dist/cjs/classes/scripts.js.map +1 -1
  9. package/dist/cjs/classes/worker.js +53 -18
  10. package/dist/cjs/classes/worker.js.map +1 -1
  11. package/dist/cjs/commands/addJobScheduler-11.lua +81 -25
  12. package/dist/cjs/commands/addRepeatableJob-2.lua +1 -1
  13. package/dist/cjs/commands/includes/addJobFromScheduler.lua +5 -3
  14. package/dist/cjs/commands/includes/getJobSchedulerEveryNextMillis.lua +28 -0
  15. package/dist/cjs/commands/includes/storeJobScheduler.lua +15 -1
  16. package/dist/cjs/commands/moveStalledJobsToWait-8.lua +14 -1
  17. package/dist/cjs/commands/promote-9.lua +0 -1
  18. package/dist/cjs/commands/updateJobScheduler-12.lua +50 -14
  19. package/dist/cjs/enums/error-code.js +2 -0
  20. package/dist/cjs/enums/error-code.js.map +1 -1
  21. package/dist/cjs/scripts/addJobScheduler-11.js +108 -25
  22. package/dist/cjs/scripts/addJobScheduler-11.js.map +1 -1
  23. package/dist/cjs/scripts/addRepeatableJob-2.js +1 -1
  24. package/dist/cjs/scripts/moveStalledJobsToWait-8.js +11 -1
  25. package/dist/cjs/scripts/moveStalledJobsToWait-8.js.map +1 -1
  26. package/dist/cjs/scripts/promote-9.js +0 -1
  27. package/dist/cjs/scripts/promote-9.js.map +1 -1
  28. package/dist/cjs/scripts/updateJobScheduler-12.js +66 -17
  29. package/dist/cjs/scripts/updateJobScheduler-12.js.map +1 -1
  30. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  31. package/dist/cjs/version.js +1 -1
  32. package/dist/esm/classes/job-scheduler.js +22 -24
  33. package/dist/esm/classes/job-scheduler.js.map +1 -1
  34. package/dist/esm/classes/job.js +19 -7
  35. package/dist/esm/classes/job.js.map +1 -1
  36. package/dist/esm/classes/queue.d.ts +1 -1
  37. package/dist/esm/classes/queue.js +1 -1
  38. package/dist/esm/classes/queue.js.map +1 -1
  39. package/dist/esm/classes/scripts.d.ts +1 -1
  40. package/dist/esm/classes/scripts.js +37 -11
  41. package/dist/esm/classes/scripts.js.map +1 -1
  42. package/dist/esm/classes/worker.js +53 -18
  43. package/dist/esm/classes/worker.js.map +1 -1
  44. package/dist/esm/commands/addJobScheduler-11.lua +81 -25
  45. package/dist/esm/commands/addRepeatableJob-2.lua +1 -1
  46. package/dist/esm/commands/includes/addJobFromScheduler.lua +5 -3
  47. package/dist/esm/commands/includes/getJobSchedulerEveryNextMillis.lua +28 -0
  48. package/dist/esm/commands/includes/storeJobScheduler.lua +15 -1
  49. package/dist/esm/commands/moveStalledJobsToWait-8.lua +14 -1
  50. package/dist/esm/commands/promote-9.lua +0 -1
  51. package/dist/esm/commands/updateJobScheduler-12.lua +50 -14
  52. package/dist/esm/enums/error-code.d.ts +3 -1
  53. package/dist/esm/enums/error-code.js +2 -0
  54. package/dist/esm/enums/error-code.js.map +1 -1
  55. package/dist/esm/interfaces/job-scheduler-json.d.ts +1 -0
  56. package/dist/esm/interfaces/repeatable-options.d.ts +1 -0
  57. package/dist/esm/scripts/addJobScheduler-11.js +108 -25
  58. package/dist/esm/scripts/addJobScheduler-11.js.map +1 -1
  59. package/dist/esm/scripts/addRepeatableJob-2.js +1 -1
  60. package/dist/esm/scripts/moveStalledJobsToWait-8.js +11 -1
  61. package/dist/esm/scripts/moveStalledJobsToWait-8.js.map +1 -1
  62. package/dist/esm/scripts/promote-9.js +0 -1
  63. package/dist/esm/scripts/promote-9.js.map +1 -1
  64. package/dist/esm/scripts/updateJobScheduler-12.js +66 -17
  65. package/dist/esm/scripts/updateJobScheduler-12.js.map +1 -1
  66. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  67. package/dist/esm/version.d.ts +1 -1
  68. package/dist/esm/version.js +1 -1
  69. package/package.json +5 -4
@@ -0,0 +1,28 @@
1
+
2
+
3
+ local function getJobSchedulerEveryNextMillis(prevMillis, every, now, offset, startDate)
4
+ local nextMillis
5
+ if not prevMillis then
6
+ if startDate then
7
+ -- Assuming startDate is passed as milliseconds from JavaScript
8
+ nextMillis = tonumber(startDate)
9
+ nextMillis = nextMillis > now and nextMillis or now
10
+ else
11
+ nextMillis = now
12
+ end
13
+ else
14
+ nextMillis = prevMillis + every
15
+ -- check if we may have missed some iterations
16
+ if nextMillis < now then
17
+ nextMillis = math.floor(now / every) * every + every + (offset or 0)
18
+ end
19
+ end
20
+
21
+ if not offset or offset == 0 then
22
+ local timeSlot = math.floor(nextMillis / every) * every;
23
+ offset = nextMillis - timeSlot;
24
+ end
25
+
26
+ -- Return a tuple nextMillis, offset
27
+ return math.floor(nextMillis), math.floor(offset)
28
+ end
@@ -21,6 +21,11 @@ local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMilli
21
21
  table.insert(optionalValues, opts['pattern'])
22
22
  end
23
23
 
24
+ if opts['startDate'] then
25
+ table.insert(optionalValues, "startDate")
26
+ table.insert(optionalValues, opts['startDate'])
27
+ end
28
+
24
29
  if opts['endDate'] then
25
30
  table.insert(optionalValues, "endDate")
26
31
  table.insert(optionalValues, opts['endDate'])
@@ -34,6 +39,12 @@ local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMilli
34
39
  if opts['offset'] then
35
40
  table.insert(optionalValues, "offset")
36
41
  table.insert(optionalValues, opts['offset'])
42
+ else
43
+ local offset = rcall("HGET", schedulerKey, "offset")
44
+ if offset then
45
+ table.insert(optionalValues, "offset")
46
+ table.insert(optionalValues, tonumber(offset))
47
+ end
37
48
  end
38
49
 
39
50
  local jsonTemplateOpts = cjson.encode(templateOpts)
@@ -47,6 +58,9 @@ local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMilli
47
58
  table.insert(optionalValues, templateData)
48
59
  end
49
60
 
61
+ table.insert(optionalValues, "ic")
62
+ table.insert(optionalValues, rcall("HGET", schedulerKey, "ic") or 1)
63
+
50
64
  rcall("DEL", schedulerKey) -- remove all attributes and then re-insert new ones
51
- rcall("HMSET", schedulerKey, "name", opts['name'], "ic", 1, unpack(optionalValues))
65
+ rcall("HMSET", schedulerKey, "name", opts['name'], unpack(optionalValues))
52
66
  end
@@ -73,10 +73,23 @@ if (#stalling > 0) then
73
73
  if (removed > 0) then
74
74
  -- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
75
75
  local stalledCount = rcall("HINCRBY", jobKey, "stc", 1)
76
- if stalledCount > maxStalledJobCount then
76
+
77
+ -- Check if this is a repeatable job by looking at job options
78
+ local jobOpts = rcall("HGET", jobKey, "opts")
79
+ local isRepeatableJob = false
80
+ if jobOpts then
81
+ local opts = cjson.decode(jobOpts)
82
+ if opts and opts["repeat"] then
83
+ isRepeatableJob = true
84
+ end
85
+ end
86
+
87
+ -- Only fail job if it exceeds stall limit AND is not a repeatable job
88
+ if stalledCount > maxStalledJobCount and not isRepeatableJob then
77
89
  local failedReason = "job stalled more than allowable limit"
78
90
  rcall("HSET", jobKey, "defa", failedReason)
79
91
  end
92
+
80
93
  moveJobToWait(metaKey, activeKey, waitKey, pausedKey, markerKey, eventStreamKey, jobId,
81
94
  "RPUSH")
82
95
 
@@ -50,7 +50,6 @@ if rcall("ZREM", KEYS[1], jobId) == 1 then
50
50
  addJobWithPriority(markerKey, KEYS[5], priority, jobId, KEYS[7], isPausedOrMaxed)
51
51
  end
52
52
 
53
- -- Emit waiting event (wait..ing@token)
54
53
  rcall("XADD", KEYS[8], "*", "event", "waiting", "jobId", jobId, "prev",
55
54
  "delayed");
56
55
 
@@ -25,31 +25,60 @@
25
25
 
26
26
  Output:
27
27
  next delayed job id - OK
28
- ]]
29
- local rcall = redis.call
28
+ ]] local rcall = redis.call
30
29
  local repeatKey = KEYS[1]
31
30
  local delayedKey = KEYS[2]
32
31
  local waitKey = KEYS[3]
33
32
  local pausedKey = KEYS[4]
34
33
  local metaKey = KEYS[5]
35
34
  local prioritizedKey = KEYS[6]
36
- local nextMillis = ARGV[1]
35
+ local nextMillis = tonumber(ARGV[1])
37
36
  local jobSchedulerId = ARGV[2]
38
- local timestamp = ARGV[5]
37
+ local timestamp = tonumber(ARGV[5])
39
38
  local prefixKey = ARGV[6]
40
39
  local producerId = ARGV[7]
40
+ local jobOpts = cmsgpack.unpack(ARGV[4])
41
41
 
42
42
  -- Includes
43
43
  --- @include "includes/addJobFromScheduler"
44
44
  --- @include "includes/getOrSetMaxEvents"
45
+ --- @include "includes/getJobSchedulerEveryNextMillis"
45
46
 
46
- local schedulerKey = repeatKey .. ":" .. jobSchedulerId
47
- local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
48
- local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
47
+ local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
49
48
 
50
49
  -- Validate that scheduler exists.
51
- local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
50
+ -- If it does not exist we should not iterate anymore.
52
51
  if prevMillis then
52
+ prevMillis = tonumber(prevMillis)
53
+
54
+ local schedulerKey = repeatKey .. ":" .. jobSchedulerId
55
+ local schedulerAttributes = rcall("HMGET", schedulerKey, "name", "data", "every", "startDate", "offset")
56
+
57
+ local every = tonumber(schedulerAttributes[3])
58
+ local now = tonumber(timestamp)
59
+
60
+ -- If every is not found in scheduler attributes, try to get it from job options
61
+ if not every and jobOpts['repeat'] and jobOpts['repeat']['every'] then
62
+ every = tonumber(jobOpts['repeat']['every'])
63
+ end
64
+
65
+ if every then
66
+ local startDate = schedulerAttributes[4]
67
+ local jobOptsOffset = jobOpts['repeat'] and jobOpts['repeat']['offset'] or 0
68
+ local offset = schedulerAttributes[5] or jobOptsOffset or 0
69
+ local newOffset
70
+
71
+ nextMillis, newOffset = getJobSchedulerEveryNextMillis(prevMillis, every, now, offset, startDate)
72
+
73
+ if not offset then
74
+ rcall("HSET", schedulerKey, "offset", newOffset)
75
+ jobOpts['repeat']['offset'] = newOffset
76
+ end
77
+ end
78
+
79
+ local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
80
+ local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
81
+
53
82
  local currentDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
54
83
 
55
84
  if producerId == currentDelayedJobId then
@@ -57,7 +86,6 @@ if prevMillis then
57
86
  local maxEvents = getOrSetMaxEvents(metaKey)
58
87
 
59
88
  if rcall("EXISTS", nextDelayedJobKey) ~= 1 then
60
- local schedulerAttributes = rcall("HMGET", schedulerKey, "name", "data")
61
89
 
62
90
  rcall("ZADD", repeatKey, nextMillis, jobSchedulerId)
63
91
  rcall("HINCRBY", schedulerKey, "ic", 1)
@@ -72,9 +100,18 @@ if prevMillis then
72
100
  rcall("HSET", schedulerKey, "data", templateData)
73
101
  end
74
102
 
75
- addJobFromScheduler(nextDelayedJobKey, nextDelayedJobId, ARGV[4], waitKey, pausedKey,
76
- KEYS[12], metaKey, prioritizedKey, KEYS[10], delayedKey, KEYS[7], eventsKey,
77
- schedulerAttributes[1], maxEvents, ARGV[5], templateData or '{}', jobSchedulerId)
103
+ local delay = nextMillis - now
104
+
105
+ -- Fast Clamp delay to minimum of 0
106
+ if delay < 0 then
107
+ delay = 0
108
+ end
109
+
110
+ jobOpts["delay"] = delay
111
+
112
+ addJobFromScheduler(nextDelayedJobKey, nextDelayedJobId, jobOpts, waitKey, pausedKey, KEYS[12], metaKey,
113
+ prioritizedKey, KEYS[10], delayedKey, KEYS[7], eventsKey, schedulerAttributes[1], maxEvents, ARGV[5],
114
+ templateData or '{}', jobSchedulerId, delay)
78
115
 
79
116
  -- TODO: remove this workaround in next breaking change
80
117
  if KEYS[11] ~= "" then
@@ -83,8 +120,7 @@ if prevMillis then
83
120
 
84
121
  return nextDelayedJobId .. "" -- convert to string
85
122
  else
86
- rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
87
- "duplicated", "jobId", nextDelayedJobId)
123
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "duplicated", "jobId", nextDelayedJobId)
88
124
  end
89
125
  end
90
126
  end
@@ -12,5 +12,7 @@ var ErrorCode;
12
12
  ErrorCode[ErrorCode["ParentJobCannotBeReplaced"] = -7] = "ParentJobCannotBeReplaced";
13
13
  ErrorCode[ErrorCode["JobBelongsToJobScheduler"] = -8] = "JobBelongsToJobScheduler";
14
14
  ErrorCode[ErrorCode["JobHasFailedChildren"] = -9] = "JobHasFailedChildren";
15
+ ErrorCode[ErrorCode["SchedulerJobIdCollision"] = -10] = "SchedulerJobIdCollision";
16
+ ErrorCode[ErrorCode["SchedulerJobSlotsBusy"] = -11] = "SchedulerJobSlotsBusy";
15
17
  })(ErrorCode || (exports.ErrorCode = ErrorCode = {}));
16
18
  //# sourceMappingURL=error-code.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"error-code.js","sourceRoot":"","sources":["../../../src/enums/error-code.ts"],"names":[],"mappings":";;;AAAA,IAAY,SAUX;AAVD,WAAY,SAAS;IACnB,wDAAgB,CAAA;IAChB,gEAAoB,CAAA;IACpB,4DAAkB,CAAA;IAClB,sEAAuB,CAAA;IACvB,oEAAsB,CAAA;IACtB,gEAAoB,CAAA;IACpB,oFAA8B,CAAA;IAC9B,kFAA6B,CAAA;IAC7B,0EAAyB,CAAA;AAC3B,CAAC,EAVW,SAAS,yBAAT,SAAS,QAUpB"}
1
+ {"version":3,"file":"error-code.js","sourceRoot":"","sources":["../../../src/enums/error-code.ts"],"names":[],"mappings":";;;AAAA,IAAY,SAYX;AAZD,WAAY,SAAS;IACnB,wDAAgB,CAAA;IAChB,gEAAoB,CAAA;IACpB,4DAAkB,CAAA;IAClB,sEAAuB,CAAA;IACvB,oEAAsB,CAAA;IACtB,gEAAoB,CAAA;IACpB,oFAA8B,CAAA;IAC9B,kFAA6B,CAAA;IAC7B,0EAAyB,CAAA;IACzB,iFAA6B,CAAA;IAC7B,6EAA2B,CAAA;AAC7B,CAAC,EAZW,SAAS,yBAAT,SAAS,QAYpB"}
@@ -19,7 +19,7 @@ const content = `--[[
19
19
  ARGV[2] msgpacked options
20
20
  [1] name
21
21
  [2] tz?
22
- [3] patten?
22
+ [3] pattern?
23
23
  [4] endDate?
24
24
  [5] every?
25
25
  ARGV[3] jobs scheduler id
@@ -43,7 +43,9 @@ local eventsKey = KEYS[9]
43
43
  local nextMillis = ARGV[1]
44
44
  local jobSchedulerId = ARGV[3]
45
45
  local templateOpts = cmsgpack.unpack(ARGV[5])
46
+ local now = tonumber(ARGV[7])
46
47
  local prefixKey = ARGV[8]
48
+ local jobOpts = cmsgpack.unpack(ARGV[6])
47
49
  -- Includes
48
50
  --[[
49
51
  Add delay marker if needed.
@@ -203,10 +205,11 @@ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed
203
205
  rcall(pushCmd, targetKey, jobId)
204
206
  addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
205
207
  end
206
- local function addJobFromScheduler(jobKey, jobId, rawOpts, waitKey, pausedKey, activeKey, metaKey,
208
+ local function addJobFromScheduler(jobKey, jobId, opts, waitKey, pausedKey, activeKey, metaKey,
207
209
  prioritizedKey, priorityCounter, delayedKey, markerKey, eventsKey, name, maxEvents, timestamp,
208
- data, jobSchedulerId)
209
- local opts = cmsgpack.unpack(rawOpts)
210
+ data, jobSchedulerId, repeatDelay)
211
+ opts['delay'] = repeatDelay
212
+ opts['jobId'] = jobId
210
213
  local delay, priority = storeJob(eventsKey, jobKey, jobId, name, data,
211
214
  opts, timestamp, nil, nil, jobSchedulerId)
212
215
  if delay ~= 0 then
@@ -377,6 +380,10 @@ local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMilli
377
380
  table.insert(optionalValues, "pattern")
378
381
  table.insert(optionalValues, opts['pattern'])
379
382
  end
383
+ if opts['startDate'] then
384
+ table.insert(optionalValues, "startDate")
385
+ table.insert(optionalValues, opts['startDate'])
386
+ end
380
387
  if opts['endDate'] then
381
388
  table.insert(optionalValues, "endDate")
382
389
  table.insert(optionalValues, opts['endDate'])
@@ -388,6 +395,12 @@ local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMilli
388
395
  if opts['offset'] then
389
396
  table.insert(optionalValues, "offset")
390
397
  table.insert(optionalValues, opts['offset'])
398
+ else
399
+ local offset = rcall("HGET", schedulerKey, "offset")
400
+ if offset then
401
+ table.insert(optionalValues, "offset")
402
+ table.insert(optionalValues, tonumber(offset))
403
+ end
391
404
  end
392
405
  local jsonTemplateOpts = cjson.encode(templateOpts)
393
406
  if jsonTemplateOpts and jsonTemplateOpts ~= '{}' then
@@ -398,15 +411,55 @@ local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMilli
398
411
  table.insert(optionalValues, "data")
399
412
  table.insert(optionalValues, templateData)
400
413
  end
414
+ table.insert(optionalValues, "ic")
415
+ table.insert(optionalValues, rcall("HGET", schedulerKey, "ic") or 1)
401
416
  rcall("DEL", schedulerKey) -- remove all attributes and then re-insert new ones
402
- rcall("HMSET", schedulerKey, "name", opts['name'], "ic", 1, unpack(optionalValues))
417
+ rcall("HMSET", schedulerKey, "name", opts['name'], unpack(optionalValues))
418
+ end
419
+ local function getJobSchedulerEveryNextMillis(prevMillis, every, now, offset, startDate)
420
+ local nextMillis
421
+ if not prevMillis then
422
+ if startDate then
423
+ -- Assuming startDate is passed as milliseconds from JavaScript
424
+ nextMillis = tonumber(startDate)
425
+ nextMillis = nextMillis > now and nextMillis or now
426
+ else
427
+ nextMillis = now
428
+ end
429
+ else
430
+ nextMillis = prevMillis + every
431
+ -- check if we may have missed some iterations
432
+ if nextMillis < now then
433
+ nextMillis = math.floor(now / every) * every + every + (offset or 0)
434
+ end
435
+ end
436
+ if not offset or offset == 0 then
437
+ local timeSlot = math.floor(nextMillis / every) * every;
438
+ offset = nextMillis - timeSlot;
439
+ end
440
+ -- Return a tuple nextMillis, offset
441
+ return math.floor(nextMillis), math.floor(offset)
403
442
  end
404
443
  -- If we are overriding a repeatable job we must delete the delayed job for
405
444
  -- the next iteration.
406
445
  local schedulerKey = repeatKey .. ":" .. jobSchedulerId
407
- local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
408
- local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
409
446
  local maxEvents = getOrSetMaxEvents(metaKey)
447
+ local templateData = ARGV[4]
448
+ local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
449
+ if prevMillis then
450
+ prevMillis = tonumber(prevMillis)
451
+ end
452
+ local schedulerOpts = cmsgpack.unpack(ARGV[2])
453
+ local every = schedulerOpts['every']
454
+ -- For backwards compatibility we also check the offset from the job itself.
455
+ -- could be removed in future major versions.
456
+ local jobOffset = jobOpts['repeat'] and jobOpts['repeat']['offset'] or 0
457
+ local offset = schedulerOpts['offset'] or jobOffset or 0
458
+ local newOffset = offset
459
+ if every then
460
+ local startDate = schedulerOpts['startDate']
461
+ nextMillis, newOffset = getJobSchedulerEveryNextMillis(prevMillis, every, now, offset, startDate)
462
+ end
410
463
  local function removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey, jobId,
411
464
  metaKey, eventsKey)
412
465
  if rcall("ZSCORE", delayedKey, jobId) then
@@ -429,33 +482,63 @@ local function removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, wai
429
482
  end
430
483
  return false
431
484
  end
432
- if rcall("EXISTS", nextDelayedJobKey) == 1 then
433
- if not removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey,
434
- nextDelayedJobId, metaKey, eventsKey) then
435
- rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
436
- "duplicated", "jobId", nextDelayedJobId)
437
- return nextDelayedJobId .. "" -- convert to string
438
- end
439
- end
440
- local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
485
+ local hadPrevJob = false
441
486
  if prevMillis then
442
487
  local currentJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
443
- local currentDelayedJobKey = schedulerKey .. ":" .. prevMillis
444
- if currentJobId ~= nextDelayedJobId and rcall("EXISTS", currentDelayedJobKey) == 1 then
445
- removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey,
488
+ local currentJobKey = schedulerKey .. ":" .. prevMillis
489
+ -- In theory it should always exist the currentJobKey if there is a prevMillis unless something has
490
+ -- gone really wrong.
491
+ if rcall("EXISTS", currentJobKey) == 1 then
492
+ hadPrevJob = removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey,
446
493
  currentJobId, metaKey, eventsKey)
447
494
  end
448
495
  end
449
- local schedulerOpts = cmsgpack.unpack(ARGV[2])
450
- storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, ARGV[4], templateOpts)
496
+ if hadPrevJob then
497
+ -- The jobs has been removed and we want to replace it, so lets use the same millis.
498
+ nextMillis = prevMillis
499
+ else
500
+ -- Special case where no job was removed, and we need to add the next iteration.
501
+ schedulerOpts['offset'] = newOffset
502
+ end
503
+ -- Check for job ID collision with existing jobs (in any state)
504
+ local jobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
505
+ local jobKey = prefixKey .. jobId
506
+ -- If there's already a job with this ID, handle the collision
507
+ if rcall("EXISTS", jobKey) == 1 then
508
+ if every then
509
+ -- For 'every' case: try next time slot to avoid collision
510
+ local nextSlotMillis = nextMillis + every
511
+ local nextSlotJobId = "repeat:" .. jobSchedulerId .. ":" .. nextSlotMillis
512
+ local nextSlotJobKey = prefixKey .. nextSlotJobId
513
+ if rcall("EXISTS", nextSlotJobKey) == 0 then
514
+ -- Next slot is free, use it
515
+ nextMillis = nextSlotMillis
516
+ jobId = nextSlotJobId
517
+ else
518
+ -- Next slot also has a job, return error code
519
+ return -11 -- SchedulerJobSlotsBusy
520
+ end
521
+ else
522
+ -- For 'pattern' case: return error code
523
+ return -10 -- SchedulerJobIdCollision
524
+ end
525
+ end
526
+ local delay = nextMillis - now
527
+ -- Fast Clamp delay to minimum of 0
528
+ if delay < 0 then
529
+ delay = 0
530
+ end
531
+ local nextJobKey = schedulerKey .. ":" .. nextMillis
532
+ -- jobId already calculated above during collision check
533
+ storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, templateData, templateOpts)
451
534
  rcall("INCR", KEYS[8])
452
- addJobFromScheduler(nextDelayedJobKey, nextDelayedJobId, ARGV[6], waitKey, pausedKey,
535
+ addJobFromScheduler(nextJobKey, jobId, jobOpts, waitKey, pausedKey,
453
536
  KEYS[11], metaKey, prioritizedKey, KEYS[10], delayedKey, KEYS[7], eventsKey,
454
- schedulerOpts['name'], maxEvents, ARGV[7], ARGV[4], jobSchedulerId)
537
+ schedulerOpts['name'], maxEvents, now, templateData, jobSchedulerId, delay)
455
538
  if ARGV[9] ~= "" then
456
- rcall("HSET", ARGV[9], "nrjid", nextDelayedJobId)
539
+ rcall("HSET", ARGV[9], "nrjid", jobId)
457
540
  end
458
- return nextDelayedJobId .. "" -- convert to string
541
+ return {jobId .. "", delay}
459
542
  `;
460
543
  exports.addJobScheduler = {
461
544
  name: 'addJobScheduler',
@@ -1 +1 @@
1
- {"version":3,"file":"addJobScheduler-11.js","sourceRoot":"","sources":["../../../src/scripts/addJobScheduler-11.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAucf,CAAC;AACW,QAAA,eAAe,GAAG;IAC7B,IAAI,EAAE,iBAAiB;IACvB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
1
+ {"version":3,"file":"addJobScheduler-11.js","sourceRoot":"","sources":["../../../src/scripts/addJobScheduler-11.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0hBf,CAAC;AACW,QAAA,eAAe,GAAG;IAC7B,IAAI,EAAE,iBAAiB;IACvB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
@@ -10,7 +10,7 @@ const content = `--[[
10
10
  ARGV[2] msgpacked options
11
11
  [1] name
12
12
  [2] tz?
13
- [3] patten?
13
+ [3] pattern?
14
14
  [4] endDate?
15
15
  [5] every?
16
16
  ARGV[3] legacy custom key TODO: remove this logic in next breaking change
@@ -144,7 +144,17 @@ if (#stalling > 0) then
144
144
  if (removed > 0) then
145
145
  -- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
146
146
  local stalledCount = rcall("HINCRBY", jobKey, "stc", 1)
147
- if stalledCount > maxStalledJobCount then
147
+ -- Check if this is a repeatable job by looking at job options
148
+ local jobOpts = rcall("HGET", jobKey, "opts")
149
+ local isRepeatableJob = false
150
+ if jobOpts then
151
+ local opts = cjson.decode(jobOpts)
152
+ if opts and opts["repeat"] then
153
+ isRepeatableJob = true
154
+ end
155
+ end
156
+ -- Only fail job if it exceeds stall limit AND is not a repeatable job
157
+ if stalledCount > maxStalledJobCount and not isRepeatableJob then
148
158
  local failedReason = "job stalled more than allowable limit"
149
159
  rcall("HSET", jobKey, "defa", failedReason)
150
160
  end
@@ -1 +1 @@
1
- {"version":3,"file":"moveStalledJobsToWait-8.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqKf,CAAC;AACW,QAAA,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"moveStalledJobsToWait-8.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+Kf,CAAC;AACW,QAAA,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -94,7 +94,6 @@ if rcall("ZREM", KEYS[1], jobId) == 1 then
94
94
  else
95
95
  addJobWithPriority(markerKey, KEYS[5], priority, jobId, KEYS[7], isPausedOrMaxed)
96
96
  end
97
- -- Emit waiting event (wait..ing@token)
98
97
  rcall("XADD", KEYS[8], "*", "event", "waiting", "jobId", jobId, "prev",
99
98
  "delayed");
100
99
  rcall("HSET", jobKey, "delay", 0)
@@ -1 +1 @@
1
- {"version":3,"file":"promote-9.js","sourceRoot":"","sources":["../../../src/scripts/promote-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqGf,CAAC;AACW,QAAA,OAAO,GAAG;IACrB,IAAI,EAAE,SAAS;IACf,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"promote-9.js","sourceRoot":"","sources":["../../../src/scripts/promote-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAoGf,CAAC;AACW,QAAA,OAAO,GAAG;IACrB,IAAI,EAAE,SAAS;IACf,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -25,19 +25,19 @@ const content = `--[[
25
25
  ARGV[7] producer id
26
26
  Output:
27
27
  next delayed job id - OK
28
- ]]
29
- local rcall = redis.call
28
+ ]] local rcall = redis.call
30
29
  local repeatKey = KEYS[1]
31
30
  local delayedKey = KEYS[2]
32
31
  local waitKey = KEYS[3]
33
32
  local pausedKey = KEYS[4]
34
33
  local metaKey = KEYS[5]
35
34
  local prioritizedKey = KEYS[6]
36
- local nextMillis = ARGV[1]
35
+ local nextMillis = tonumber(ARGV[1])
37
36
  local jobSchedulerId = ARGV[2]
38
- local timestamp = ARGV[5]
37
+ local timestamp = tonumber(ARGV[5])
39
38
  local prefixKey = ARGV[6]
40
39
  local producerId = ARGV[7]
40
+ local jobOpts = cmsgpack.unpack(ARGV[4])
41
41
  -- Includes
42
42
  --[[
43
43
  Add delay marker if needed.
@@ -197,10 +197,11 @@ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed
197
197
  rcall(pushCmd, targetKey, jobId)
198
198
  addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
199
199
  end
200
- local function addJobFromScheduler(jobKey, jobId, rawOpts, waitKey, pausedKey, activeKey, metaKey,
200
+ local function addJobFromScheduler(jobKey, jobId, opts, waitKey, pausedKey, activeKey, metaKey,
201
201
  prioritizedKey, priorityCounter, delayedKey, markerKey, eventsKey, name, maxEvents, timestamp,
202
- data, jobSchedulerId)
203
- local opts = cmsgpack.unpack(rawOpts)
202
+ data, jobSchedulerId, repeatDelay)
203
+ opts['delay'] = repeatDelay
204
+ opts['jobId'] = jobId
204
205
  local delay, priority = storeJob(eventsKey, jobKey, jobId, name, data,
205
206
  opts, timestamp, nil, nil, jobSchedulerId)
206
207
  if delay ~= 0 then
@@ -230,18 +231,61 @@ local function getOrSetMaxEvents(metaKey)
230
231
  end
231
232
  return maxEvents
232
233
  end
233
- local schedulerKey = repeatKey .. ":" .. jobSchedulerId
234
- local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
235
- local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
236
- -- Validate that scheduler exists.
234
+ local function getJobSchedulerEveryNextMillis(prevMillis, every, now, offset, startDate)
235
+ local nextMillis
236
+ if not prevMillis then
237
+ if startDate then
238
+ -- Assuming startDate is passed as milliseconds from JavaScript
239
+ nextMillis = tonumber(startDate)
240
+ nextMillis = nextMillis > now and nextMillis or now
241
+ else
242
+ nextMillis = now
243
+ end
244
+ else
245
+ nextMillis = prevMillis + every
246
+ -- check if we may have missed some iterations
247
+ if nextMillis < now then
248
+ nextMillis = math.floor(now / every) * every + every + (offset or 0)
249
+ end
250
+ end
251
+ if not offset or offset == 0 then
252
+ local timeSlot = math.floor(nextMillis / every) * every;
253
+ offset = nextMillis - timeSlot;
254
+ end
255
+ -- Return a tuple nextMillis, offset
256
+ return math.floor(nextMillis), math.floor(offset)
257
+ end
237
258
  local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
259
+ -- Validate that scheduler exists.
260
+ -- If it does not exist we should not iterate anymore.
238
261
  if prevMillis then
262
+ prevMillis = tonumber(prevMillis)
263
+ local schedulerKey = repeatKey .. ":" .. jobSchedulerId
264
+ local schedulerAttributes = rcall("HMGET", schedulerKey, "name", "data", "every", "startDate", "offset")
265
+ local every = tonumber(schedulerAttributes[3])
266
+ local now = tonumber(timestamp)
267
+ -- If every is not found in scheduler attributes, try to get it from job options
268
+ if not every and jobOpts['repeat'] and jobOpts['repeat']['every'] then
269
+ every = tonumber(jobOpts['repeat']['every'])
270
+ end
271
+ if every then
272
+ local startDate = schedulerAttributes[4]
273
+ local jobOptsOffset = jobOpts['repeat'] and jobOpts['repeat']['offset'] or 0
274
+ local offset = schedulerAttributes[5] or jobOptsOffset or 0
275
+ local newOffset
276
+ nextMillis, newOffset = getJobSchedulerEveryNextMillis(prevMillis, every, now, offset, startDate)
277
+ if not offset then
278
+ rcall("HSET", schedulerKey, "offset", newOffset)
279
+ jobOpts['repeat']['offset'] = newOffset
280
+ end
281
+ end
282
+ local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
283
+ local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
239
284
  local currentDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
240
285
  if producerId == currentDelayedJobId then
241
286
  local eventsKey = KEYS[9]
242
287
  local maxEvents = getOrSetMaxEvents(metaKey)
243
288
  if rcall("EXISTS", nextDelayedJobKey) ~= 1 then
244
- local schedulerAttributes = rcall("HMGET", schedulerKey, "name", "data")
245
289
  rcall("ZADD", repeatKey, nextMillis, jobSchedulerId)
246
290
  rcall("HINCRBY", schedulerKey, "ic", 1)
247
291
  rcall("INCR", KEYS[8])
@@ -251,17 +295,22 @@ if prevMillis then
251
295
  if templateData and templateData ~= '{}' then
252
296
  rcall("HSET", schedulerKey, "data", templateData)
253
297
  end
254
- addJobFromScheduler(nextDelayedJobKey, nextDelayedJobId, ARGV[4], waitKey, pausedKey,
255
- KEYS[12], metaKey, prioritizedKey, KEYS[10], delayedKey, KEYS[7], eventsKey,
256
- schedulerAttributes[1], maxEvents, ARGV[5], templateData or '{}', jobSchedulerId)
298
+ local delay = nextMillis - now
299
+ -- Fast Clamp delay to minimum of 0
300
+ if delay < 0 then
301
+ delay = 0
302
+ end
303
+ jobOpts["delay"] = delay
304
+ addJobFromScheduler(nextDelayedJobKey, nextDelayedJobId, jobOpts, waitKey, pausedKey, KEYS[12], metaKey,
305
+ prioritizedKey, KEYS[10], delayedKey, KEYS[7], eventsKey, schedulerAttributes[1], maxEvents, ARGV[5],
306
+ templateData or '{}', jobSchedulerId, delay)
257
307
  -- TODO: remove this workaround in next breaking change
258
308
  if KEYS[11] ~= "" then
259
309
  rcall("HSET", KEYS[11], "nrjid", nextDelayedJobId)
260
310
  end
261
311
  return nextDelayedJobId .. "" -- convert to string
262
312
  else
263
- rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
264
- "duplicated", "jobId", nextDelayedJobId)
313
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "duplicated", "jobId", nextDelayedJobId)
265
314
  end
266
315
  end
267
316
  end
@@ -1 +1 @@
1
- {"version":3,"file":"updateJobScheduler-12.js","sourceRoot":"","sources":["../../../src/scripts/updateJobScheduler-12.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAwQf,CAAC;AACW,QAAA,kBAAkB,GAAG;IAChC,IAAI,EAAE,oBAAoB;IAC1B,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
1
+ {"version":3,"file":"updateJobScheduler-12.js","sourceRoot":"","sources":["../../../src/scripts/updateJobScheduler-12.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAyTf,CAAC;AACW,QAAA,kBAAkB,GAAG;IAChC,IAAI,EAAE,oBAAoB;IAC1B,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}