bullmq 5.58.7 → 5.58.9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. package/dist/cjs/classes/job-scheduler.js +22 -24
  2. package/dist/cjs/classes/job-scheduler.js.map +1 -1
  3. package/dist/cjs/classes/job.js +6 -2
  4. package/dist/cjs/classes/job.js.map +1 -1
  5. package/dist/cjs/classes/queue-getters.js +6 -19
  6. package/dist/cjs/classes/queue-getters.js.map +1 -1
  7. package/dist/cjs/classes/queue.js +1 -1
  8. package/dist/cjs/classes/queue.js.map +1 -1
  9. package/dist/cjs/classes/scripts.js +47 -11
  10. package/dist/cjs/classes/scripts.js.map +1 -1
  11. package/dist/cjs/classes/worker.js +53 -18
  12. package/dist/cjs/classes/worker.js.map +1 -1
  13. package/dist/cjs/commands/addJobScheduler-11.lua +81 -25
  14. package/dist/cjs/commands/addRepeatableJob-2.lua +1 -1
  15. package/dist/cjs/commands/getMetrics-2.lua +19 -0
  16. package/dist/cjs/commands/includes/addJobFromScheduler.lua +5 -3
  17. package/dist/cjs/commands/includes/getJobSchedulerEveryNextMillis.lua +28 -0
  18. package/dist/cjs/commands/includes/storeJobScheduler.lua +15 -1
  19. package/dist/cjs/commands/moveStalledJobsToWait-8.lua +14 -1
  20. package/dist/cjs/commands/updateJobScheduler-12.lua +50 -14
  21. package/dist/cjs/enums/error-code.js +2 -0
  22. package/dist/cjs/enums/error-code.js.map +1 -1
  23. package/dist/cjs/scripts/addJobScheduler-11.js +108 -25
  24. package/dist/cjs/scripts/addJobScheduler-11.js.map +1 -1
  25. package/dist/cjs/scripts/addRepeatableJob-2.js +1 -1
  26. package/dist/cjs/scripts/getMetrics-2.js +25 -0
  27. package/dist/cjs/scripts/getMetrics-2.js.map +1 -0
  28. package/dist/cjs/scripts/index.js +1 -0
  29. package/dist/cjs/scripts/index.js.map +1 -1
  30. package/dist/cjs/scripts/moveStalledJobsToWait-8.js +11 -1
  31. package/dist/cjs/scripts/moveStalledJobsToWait-8.js.map +1 -1
  32. package/dist/cjs/scripts/updateJobScheduler-12.js +66 -17
  33. package/dist/cjs/scripts/updateJobScheduler-12.js.map +1 -1
  34. package/dist/cjs/tsconfig-cjs.tsbuildinfo +1 -1
  35. package/dist/cjs/version.js +1 -1
  36. package/dist/esm/classes/job-scheduler.js +22 -24
  37. package/dist/esm/classes/job-scheduler.js.map +1 -1
  38. package/dist/esm/classes/job.js +6 -2
  39. package/dist/esm/classes/job.js.map +1 -1
  40. package/dist/esm/classes/queue-getters.js +6 -19
  41. package/dist/esm/classes/queue-getters.js.map +1 -1
  42. package/dist/esm/classes/queue.d.ts +1 -1
  43. package/dist/esm/classes/queue.js +1 -1
  44. package/dist/esm/classes/queue.js.map +1 -1
  45. package/dist/esm/classes/scripts.d.ts +2 -1
  46. package/dist/esm/classes/scripts.js +47 -11
  47. package/dist/esm/classes/scripts.js.map +1 -1
  48. package/dist/esm/classes/worker.js +53 -18
  49. package/dist/esm/classes/worker.js.map +1 -1
  50. package/dist/esm/commands/addJobScheduler-11.lua +81 -25
  51. package/dist/esm/commands/addRepeatableJob-2.lua +1 -1
  52. package/dist/esm/commands/getMetrics-2.lua +19 -0
  53. package/dist/esm/commands/includes/addJobFromScheduler.lua +5 -3
  54. package/dist/esm/commands/includes/getJobSchedulerEveryNextMillis.lua +28 -0
  55. package/dist/esm/commands/includes/storeJobScheduler.lua +15 -1
  56. package/dist/esm/commands/moveStalledJobsToWait-8.lua +14 -1
  57. package/dist/esm/commands/updateJobScheduler-12.lua +50 -14
  58. package/dist/esm/enums/error-code.d.ts +3 -1
  59. package/dist/esm/enums/error-code.js +2 -0
  60. package/dist/esm/enums/error-code.js.map +1 -1
  61. package/dist/esm/interfaces/job-scheduler-json.d.ts +1 -0
  62. package/dist/esm/interfaces/repeatable-options.d.ts +1 -0
  63. package/dist/esm/scripts/addJobScheduler-11.js +108 -25
  64. package/dist/esm/scripts/addJobScheduler-11.js.map +1 -1
  65. package/dist/esm/scripts/addRepeatableJob-2.js +1 -1
  66. package/dist/esm/scripts/getMetrics-2.d.ts +5 -0
  67. package/dist/esm/scripts/getMetrics-2.js +22 -0
  68. package/dist/esm/scripts/getMetrics-2.js.map +1 -0
  69. package/dist/esm/scripts/index.d.ts +1 -0
  70. package/dist/esm/scripts/index.js +1 -0
  71. package/dist/esm/scripts/index.js.map +1 -1
  72. package/dist/esm/scripts/moveStalledJobsToWait-8.js +11 -1
  73. package/dist/esm/scripts/moveStalledJobsToWait-8.js.map +1 -1
  74. package/dist/esm/scripts/updateJobScheduler-12.js +66 -17
  75. package/dist/esm/scripts/updateJobScheduler-12.js.map +1 -1
  76. package/dist/esm/tsconfig.tsbuildinfo +1 -1
  77. package/dist/esm/version.d.ts +1 -1
  78. package/dist/esm/version.js +1 -1
  79. package/package.json +5 -5
@@ -10,10 +10,12 @@
10
10
  --- @include "getTargetQueueList"
11
11
  --- @include "addJobInTargetList"
12
12
 
13
- local function addJobFromScheduler(jobKey, jobId, rawOpts, waitKey, pausedKey, activeKey, metaKey,
13
+ local function addJobFromScheduler(jobKey, jobId, opts, waitKey, pausedKey, activeKey, metaKey,
14
14
  prioritizedKey, priorityCounter, delayedKey, markerKey, eventsKey, name, maxEvents, timestamp,
15
- data, jobSchedulerId)
16
- local opts = cmsgpack.unpack(rawOpts)
15
+ data, jobSchedulerId, repeatDelay)
16
+
17
+ opts['delay'] = repeatDelay
18
+ opts['jobId'] = jobId
17
19
 
18
20
  local delay, priority = storeJob(eventsKey, jobKey, jobId, name, data,
19
21
  opts, timestamp, nil, nil, jobSchedulerId)
@@ -0,0 +1,28 @@
1
+
2
+
3
+ local function getJobSchedulerEveryNextMillis(prevMillis, every, now, offset, startDate)
4
+ local nextMillis
5
+ if not prevMillis then
6
+ if startDate then
7
+ -- Assuming startDate is passed as milliseconds from JavaScript
8
+ nextMillis = tonumber(startDate)
9
+ nextMillis = nextMillis > now and nextMillis or now
10
+ else
11
+ nextMillis = now
12
+ end
13
+ else
14
+ nextMillis = prevMillis + every
15
+ -- check if we may have missed some iterations
16
+ if nextMillis < now then
17
+ nextMillis = math.floor(now / every) * every + every + (offset or 0)
18
+ end
19
+ end
20
+
21
+ if not offset or offset == 0 then
22
+ local timeSlot = math.floor(nextMillis / every) * every;
23
+ offset = nextMillis - timeSlot;
24
+ end
25
+
26
+ -- Return a tuple nextMillis, offset
27
+ return math.floor(nextMillis), math.floor(offset)
28
+ end
@@ -21,6 +21,11 @@ local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMilli
21
21
  table.insert(optionalValues, opts['pattern'])
22
22
  end
23
23
 
24
+ if opts['startDate'] then
25
+ table.insert(optionalValues, "startDate")
26
+ table.insert(optionalValues, opts['startDate'])
27
+ end
28
+
24
29
  if opts['endDate'] then
25
30
  table.insert(optionalValues, "endDate")
26
31
  table.insert(optionalValues, opts['endDate'])
@@ -34,6 +39,12 @@ local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMilli
34
39
  if opts['offset'] then
35
40
  table.insert(optionalValues, "offset")
36
41
  table.insert(optionalValues, opts['offset'])
42
+ else
43
+ local offset = rcall("HGET", schedulerKey, "offset")
44
+ if offset then
45
+ table.insert(optionalValues, "offset")
46
+ table.insert(optionalValues, tonumber(offset))
47
+ end
37
48
  end
38
49
 
39
50
  local jsonTemplateOpts = cjson.encode(templateOpts)
@@ -47,6 +58,9 @@ local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMilli
47
58
  table.insert(optionalValues, templateData)
48
59
  end
49
60
 
61
+ table.insert(optionalValues, "ic")
62
+ table.insert(optionalValues, rcall("HGET", schedulerKey, "ic") or 1)
63
+
50
64
  rcall("DEL", schedulerKey) -- remove all attributes and then re-insert new ones
51
- rcall("HMSET", schedulerKey, "name", opts['name'], "ic", 1, unpack(optionalValues))
65
+ rcall("HMSET", schedulerKey, "name", opts['name'], unpack(optionalValues))
52
66
  end
@@ -73,10 +73,23 @@ if (#stalling > 0) then
73
73
  if (removed > 0) then
74
74
  -- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
75
75
  local stalledCount = rcall("HINCRBY", jobKey, "stc", 1)
76
- if stalledCount > maxStalledJobCount then
76
+
77
+ -- Check if this is a repeatable job by looking at job options
78
+ local jobOpts = rcall("HGET", jobKey, "opts")
79
+ local isRepeatableJob = false
80
+ if jobOpts then
81
+ local opts = cjson.decode(jobOpts)
82
+ if opts and opts["repeat"] then
83
+ isRepeatableJob = true
84
+ end
85
+ end
86
+
87
+ -- Only fail job if it exceeds stall limit AND is not a repeatable job
88
+ if stalledCount > maxStalledJobCount and not isRepeatableJob then
77
89
  local failedReason = "job stalled more than allowable limit"
78
90
  rcall("HSET", jobKey, "defa", failedReason)
79
91
  end
92
+
80
93
  moveJobToWait(metaKey, activeKey, waitKey, pausedKey, markerKey, eventStreamKey, jobId,
81
94
  "RPUSH")
82
95
 
@@ -25,31 +25,60 @@
25
25
 
26
26
  Output:
27
27
  next delayed job id - OK
28
- ]]
29
- local rcall = redis.call
28
+ ]] local rcall = redis.call
30
29
  local repeatKey = KEYS[1]
31
30
  local delayedKey = KEYS[2]
32
31
  local waitKey = KEYS[3]
33
32
  local pausedKey = KEYS[4]
34
33
  local metaKey = KEYS[5]
35
34
  local prioritizedKey = KEYS[6]
36
- local nextMillis = ARGV[1]
35
+ local nextMillis = tonumber(ARGV[1])
37
36
  local jobSchedulerId = ARGV[2]
38
- local timestamp = ARGV[5]
37
+ local timestamp = tonumber(ARGV[5])
39
38
  local prefixKey = ARGV[6]
40
39
  local producerId = ARGV[7]
40
+ local jobOpts = cmsgpack.unpack(ARGV[4])
41
41
 
42
42
  -- Includes
43
43
  --- @include "includes/addJobFromScheduler"
44
44
  --- @include "includes/getOrSetMaxEvents"
45
+ --- @include "includes/getJobSchedulerEveryNextMillis"
45
46
 
46
- local schedulerKey = repeatKey .. ":" .. jobSchedulerId
47
- local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
48
- local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
47
+ local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
49
48
 
50
49
  -- Validate that scheduler exists.
51
- local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
50
+ -- If it does not exist we should not iterate anymore.
52
51
  if prevMillis then
52
+ prevMillis = tonumber(prevMillis)
53
+
54
+ local schedulerKey = repeatKey .. ":" .. jobSchedulerId
55
+ local schedulerAttributes = rcall("HMGET", schedulerKey, "name", "data", "every", "startDate", "offset")
56
+
57
+ local every = tonumber(schedulerAttributes[3])
58
+ local now = tonumber(timestamp)
59
+
60
+ -- If every is not found in scheduler attributes, try to get it from job options
61
+ if not every and jobOpts['repeat'] and jobOpts['repeat']['every'] then
62
+ every = tonumber(jobOpts['repeat']['every'])
63
+ end
64
+
65
+ if every then
66
+ local startDate = schedulerAttributes[4]
67
+ local jobOptsOffset = jobOpts['repeat'] and jobOpts['repeat']['offset'] or 0
68
+ local offset = schedulerAttributes[5] or jobOptsOffset or 0
69
+ local newOffset
70
+
71
+ nextMillis, newOffset = getJobSchedulerEveryNextMillis(prevMillis, every, now, offset, startDate)
72
+
73
+ if not offset then
74
+ rcall("HSET", schedulerKey, "offset", newOffset)
75
+ jobOpts['repeat']['offset'] = newOffset
76
+ end
77
+ end
78
+
79
+ local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
80
+ local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
81
+
53
82
  local currentDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
54
83
 
55
84
  if producerId == currentDelayedJobId then
@@ -57,7 +86,6 @@ if prevMillis then
57
86
  local maxEvents = getOrSetMaxEvents(metaKey)
58
87
 
59
88
  if rcall("EXISTS", nextDelayedJobKey) ~= 1 then
60
- local schedulerAttributes = rcall("HMGET", schedulerKey, "name", "data")
61
89
 
62
90
  rcall("ZADD", repeatKey, nextMillis, jobSchedulerId)
63
91
  rcall("HINCRBY", schedulerKey, "ic", 1)
@@ -72,9 +100,18 @@ if prevMillis then
72
100
  rcall("HSET", schedulerKey, "data", templateData)
73
101
  end
74
102
 
75
- addJobFromScheduler(nextDelayedJobKey, nextDelayedJobId, ARGV[4], waitKey, pausedKey,
76
- KEYS[12], metaKey, prioritizedKey, KEYS[10], delayedKey, KEYS[7], eventsKey,
77
- schedulerAttributes[1], maxEvents, ARGV[5], templateData or '{}', jobSchedulerId)
103
+ local delay = nextMillis - now
104
+
105
+ -- Fast Clamp delay to minimum of 0
106
+ if delay < 0 then
107
+ delay = 0
108
+ end
109
+
110
+ jobOpts["delay"] = delay
111
+
112
+ addJobFromScheduler(nextDelayedJobKey, nextDelayedJobId, jobOpts, waitKey, pausedKey, KEYS[12], metaKey,
113
+ prioritizedKey, KEYS[10], delayedKey, KEYS[7], eventsKey, schedulerAttributes[1], maxEvents, ARGV[5],
114
+ templateData or '{}', jobSchedulerId, delay)
78
115
 
79
116
  -- TODO: remove this workaround in next breaking change
80
117
  if KEYS[11] ~= "" then
@@ -83,8 +120,7 @@ if prevMillis then
83
120
 
84
121
  return nextDelayedJobId .. "" -- convert to string
85
122
  else
86
- rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
87
- "duplicated", "jobId", nextDelayedJobId)
123
+ rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event", "duplicated", "jobId", nextDelayedJobId)
88
124
  end
89
125
  end
90
126
  end
@@ -7,5 +7,7 @@ export declare enum ErrorCode {
7
7
  JobLockMismatch = -6,
8
8
  ParentJobCannotBeReplaced = -7,
9
9
  JobBelongsToJobScheduler = -8,
10
- JobHasFailedChildren = -9
10
+ JobHasFailedChildren = -9,
11
+ SchedulerJobIdCollision = -10,
12
+ SchedulerJobSlotsBusy = -11
11
13
  }
@@ -9,5 +9,7 @@ export var ErrorCode;
9
9
  ErrorCode[ErrorCode["ParentJobCannotBeReplaced"] = -7] = "ParentJobCannotBeReplaced";
10
10
  ErrorCode[ErrorCode["JobBelongsToJobScheduler"] = -8] = "JobBelongsToJobScheduler";
11
11
  ErrorCode[ErrorCode["JobHasFailedChildren"] = -9] = "JobHasFailedChildren";
12
+ ErrorCode[ErrorCode["SchedulerJobIdCollision"] = -10] = "SchedulerJobIdCollision";
13
+ ErrorCode[ErrorCode["SchedulerJobSlotsBusy"] = -11] = "SchedulerJobSlotsBusy";
12
14
  })(ErrorCode || (ErrorCode = {}));
13
15
  //# sourceMappingURL=error-code.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"error-code.js","sourceRoot":"","sources":["../../../src/enums/error-code.ts"],"names":[],"mappings":"AAAA,MAAM,CAAN,IAAY,SAUX;AAVD,WAAY,SAAS;IACnB,wDAAgB,CAAA;IAChB,gEAAoB,CAAA;IACpB,4DAAkB,CAAA;IAClB,sEAAuB,CAAA;IACvB,oEAAsB,CAAA;IACtB,gEAAoB,CAAA;IACpB,oFAA8B,CAAA;IAC9B,kFAA6B,CAAA;IAC7B,0EAAyB,CAAA;AAC3B,CAAC,EAVW,SAAS,KAAT,SAAS,QAUpB"}
1
+ {"version":3,"file":"error-code.js","sourceRoot":"","sources":["../../../src/enums/error-code.ts"],"names":[],"mappings":"AAAA,MAAM,CAAN,IAAY,SAYX;AAZD,WAAY,SAAS;IACnB,wDAAgB,CAAA;IAChB,gEAAoB,CAAA;IACpB,4DAAkB,CAAA;IAClB,sEAAuB,CAAA;IACvB,oEAAsB,CAAA;IACtB,gEAAoB,CAAA;IACpB,oFAA8B,CAAA;IAC9B,kFAA6B,CAAA;IAC7B,0EAAyB,CAAA;IACzB,iFAA6B,CAAA;IAC7B,6EAA2B,CAAA;AAC7B,CAAC,EAZW,SAAS,KAAT,SAAS,QAYpB"}
@@ -9,6 +9,7 @@ export interface JobSchedulerJson<D = any> {
9
9
  id?: string | null;
10
10
  iterationCount?: number;
11
11
  limit?: number;
12
+ startDate?: number;
12
13
  endDate?: number;
13
14
  tz?: string;
14
15
  pattern?: string;
@@ -1,5 +1,6 @@
1
1
  export type RepeatableOptions = {
2
2
  name: string;
3
+ startDate?: number;
3
4
  endDate?: number;
4
5
  tz?: string;
5
6
  limit?: number;
@@ -16,7 +16,7 @@ const content = `--[[
16
16
  ARGV[2] msgpacked options
17
17
  [1] name
18
18
  [2] tz?
19
- [3] patten?
19
+ [3] pattern?
20
20
  [4] endDate?
21
21
  [5] every?
22
22
  ARGV[3] jobs scheduler id
@@ -40,7 +40,9 @@ local eventsKey = KEYS[9]
40
40
  local nextMillis = ARGV[1]
41
41
  local jobSchedulerId = ARGV[3]
42
42
  local templateOpts = cmsgpack.unpack(ARGV[5])
43
+ local now = tonumber(ARGV[7])
43
44
  local prefixKey = ARGV[8]
45
+ local jobOpts = cmsgpack.unpack(ARGV[6])
44
46
  -- Includes
45
47
  --[[
46
48
  Add delay marker if needed.
@@ -200,10 +202,11 @@ local function addJobInTargetList(targetKey, markerKey, pushCmd, isPausedOrMaxed
200
202
  rcall(pushCmd, targetKey, jobId)
201
203
  addBaseMarkerIfNeeded(markerKey, isPausedOrMaxed)
202
204
  end
203
- local function addJobFromScheduler(jobKey, jobId, rawOpts, waitKey, pausedKey, activeKey, metaKey,
205
+ local function addJobFromScheduler(jobKey, jobId, opts, waitKey, pausedKey, activeKey, metaKey,
204
206
  prioritizedKey, priorityCounter, delayedKey, markerKey, eventsKey, name, maxEvents, timestamp,
205
- data, jobSchedulerId)
206
- local opts = cmsgpack.unpack(rawOpts)
207
+ data, jobSchedulerId, repeatDelay)
208
+ opts['delay'] = repeatDelay
209
+ opts['jobId'] = jobId
207
210
  local delay, priority = storeJob(eventsKey, jobKey, jobId, name, data,
208
211
  opts, timestamp, nil, nil, jobSchedulerId)
209
212
  if delay ~= 0 then
@@ -374,6 +377,10 @@ local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMilli
374
377
  table.insert(optionalValues, "pattern")
375
378
  table.insert(optionalValues, opts['pattern'])
376
379
  end
380
+ if opts['startDate'] then
381
+ table.insert(optionalValues, "startDate")
382
+ table.insert(optionalValues, opts['startDate'])
383
+ end
377
384
  if opts['endDate'] then
378
385
  table.insert(optionalValues, "endDate")
379
386
  table.insert(optionalValues, opts['endDate'])
@@ -385,6 +392,12 @@ local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMilli
385
392
  if opts['offset'] then
386
393
  table.insert(optionalValues, "offset")
387
394
  table.insert(optionalValues, opts['offset'])
395
+ else
396
+ local offset = rcall("HGET", schedulerKey, "offset")
397
+ if offset then
398
+ table.insert(optionalValues, "offset")
399
+ table.insert(optionalValues, tonumber(offset))
400
+ end
388
401
  end
389
402
  local jsonTemplateOpts = cjson.encode(templateOpts)
390
403
  if jsonTemplateOpts and jsonTemplateOpts ~= '{}' then
@@ -395,15 +408,55 @@ local function storeJobScheduler(schedulerId, schedulerKey, repeatKey, nextMilli
395
408
  table.insert(optionalValues, "data")
396
409
  table.insert(optionalValues, templateData)
397
410
  end
411
+ table.insert(optionalValues, "ic")
412
+ table.insert(optionalValues, rcall("HGET", schedulerKey, "ic") or 1)
398
413
  rcall("DEL", schedulerKey) -- remove all attributes and then re-insert new ones
399
- rcall("HMSET", schedulerKey, "name", opts['name'], "ic", 1, unpack(optionalValues))
414
+ rcall("HMSET", schedulerKey, "name", opts['name'], unpack(optionalValues))
415
+ end
416
+ local function getJobSchedulerEveryNextMillis(prevMillis, every, now, offset, startDate)
417
+ local nextMillis
418
+ if not prevMillis then
419
+ if startDate then
420
+ -- Assuming startDate is passed as milliseconds from JavaScript
421
+ nextMillis = tonumber(startDate)
422
+ nextMillis = nextMillis > now and nextMillis or now
423
+ else
424
+ nextMillis = now
425
+ end
426
+ else
427
+ nextMillis = prevMillis + every
428
+ -- check if we may have missed some iterations
429
+ if nextMillis < now then
430
+ nextMillis = math.floor(now / every) * every + every + (offset or 0)
431
+ end
432
+ end
433
+ if not offset or offset == 0 then
434
+ local timeSlot = math.floor(nextMillis / every) * every;
435
+ offset = nextMillis - timeSlot;
436
+ end
437
+ -- Return a tuple nextMillis, offset
438
+ return math.floor(nextMillis), math.floor(offset)
400
439
  end
401
440
  -- If we are overriding a repeatable job we must delete the delayed job for
402
441
  -- the next iteration.
403
442
  local schedulerKey = repeatKey .. ":" .. jobSchedulerId
404
- local nextDelayedJobKey = schedulerKey .. ":" .. nextMillis
405
- local nextDelayedJobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
406
443
  local maxEvents = getOrSetMaxEvents(metaKey)
444
+ local templateData = ARGV[4]
445
+ local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
446
+ if prevMillis then
447
+ prevMillis = tonumber(prevMillis)
448
+ end
449
+ local schedulerOpts = cmsgpack.unpack(ARGV[2])
450
+ local every = schedulerOpts['every']
451
+ -- For backwards compatibility we also check the offset from the job itself.
452
+ -- could be removed in future major versions.
453
+ local jobOffset = jobOpts['repeat'] and jobOpts['repeat']['offset'] or 0
454
+ local offset = schedulerOpts['offset'] or jobOffset or 0
455
+ local newOffset = offset
456
+ if every then
457
+ local startDate = schedulerOpts['startDate']
458
+ nextMillis, newOffset = getJobSchedulerEveryNextMillis(prevMillis, every, now, offset, startDate)
459
+ end
407
460
  local function removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey, jobId,
408
461
  metaKey, eventsKey)
409
462
  if rcall("ZSCORE", delayedKey, jobId) then
@@ -426,33 +479,63 @@ local function removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, wai
426
479
  end
427
480
  return false
428
481
  end
429
- if rcall("EXISTS", nextDelayedJobKey) == 1 then
430
- if not removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey,
431
- nextDelayedJobId, metaKey, eventsKey) then
432
- rcall("XADD", eventsKey, "MAXLEN", "~", maxEvents, "*", "event",
433
- "duplicated", "jobId", nextDelayedJobId)
434
- return nextDelayedJobId .. "" -- convert to string
435
- end
436
- end
437
- local prevMillis = rcall("ZSCORE", repeatKey, jobSchedulerId)
482
+ local hadPrevJob = false
438
483
  if prevMillis then
439
484
  local currentJobId = "repeat:" .. jobSchedulerId .. ":" .. prevMillis
440
- local currentDelayedJobKey = schedulerKey .. ":" .. prevMillis
441
- if currentJobId ~= nextDelayedJobId and rcall("EXISTS", currentDelayedJobKey) == 1 then
442
- removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey,
485
+ local currentJobKey = schedulerKey .. ":" .. prevMillis
486
+ -- In theory it should always exist the currentJobKey if there is a prevMillis unless something has
487
+ -- gone really wrong.
488
+ if rcall("EXISTS", currentJobKey) == 1 then
489
+ hadPrevJob = removeJobFromScheduler(prefixKey, delayedKey, prioritizedKey, waitKey, pausedKey,
443
490
  currentJobId, metaKey, eventsKey)
444
491
  end
445
492
  end
446
- local schedulerOpts = cmsgpack.unpack(ARGV[2])
447
- storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, ARGV[4], templateOpts)
493
+ if hadPrevJob then
494
+ -- The jobs has been removed and we want to replace it, so lets use the same millis.
495
+ nextMillis = prevMillis
496
+ else
497
+ -- Special case where no job was removed, and we need to add the next iteration.
498
+ schedulerOpts['offset'] = newOffset
499
+ end
500
+ -- Check for job ID collision with existing jobs (in any state)
501
+ local jobId = "repeat:" .. jobSchedulerId .. ":" .. nextMillis
502
+ local jobKey = prefixKey .. jobId
503
+ -- If there's already a job with this ID, handle the collision
504
+ if rcall("EXISTS", jobKey) == 1 then
505
+ if every then
506
+ -- For 'every' case: try next time slot to avoid collision
507
+ local nextSlotMillis = nextMillis + every
508
+ local nextSlotJobId = "repeat:" .. jobSchedulerId .. ":" .. nextSlotMillis
509
+ local nextSlotJobKey = prefixKey .. nextSlotJobId
510
+ if rcall("EXISTS", nextSlotJobKey) == 0 then
511
+ -- Next slot is free, use it
512
+ nextMillis = nextSlotMillis
513
+ jobId = nextSlotJobId
514
+ else
515
+ -- Next slot also has a job, return error code
516
+ return -11 -- SchedulerJobSlotsBusy
517
+ end
518
+ else
519
+ -- For 'pattern' case: return error code
520
+ return -10 -- SchedulerJobIdCollision
521
+ end
522
+ end
523
+ local delay = nextMillis - now
524
+ -- Fast Clamp delay to minimum of 0
525
+ if delay < 0 then
526
+ delay = 0
527
+ end
528
+ local nextJobKey = schedulerKey .. ":" .. nextMillis
529
+ -- jobId already calculated above during collision check
530
+ storeJobScheduler(jobSchedulerId, schedulerKey, repeatKey, nextMillis, schedulerOpts, templateData, templateOpts)
448
531
  rcall("INCR", KEYS[8])
449
- addJobFromScheduler(nextDelayedJobKey, nextDelayedJobId, ARGV[6], waitKey, pausedKey,
532
+ addJobFromScheduler(nextJobKey, jobId, jobOpts, waitKey, pausedKey,
450
533
  KEYS[11], metaKey, prioritizedKey, KEYS[10], delayedKey, KEYS[7], eventsKey,
451
- schedulerOpts['name'], maxEvents, ARGV[7], ARGV[4], jobSchedulerId)
534
+ schedulerOpts['name'], maxEvents, now, templateData, jobSchedulerId, delay)
452
535
  if ARGV[9] ~= "" then
453
- rcall("HSET", ARGV[9], "nrjid", nextDelayedJobId)
536
+ rcall("HSET", ARGV[9], "nrjid", jobId)
454
537
  end
455
- return nextDelayedJobId .. "" -- convert to string
538
+ return {jobId .. "", delay}
456
539
  `;
457
540
  export const addJobScheduler = {
458
541
  name: 'addJobScheduler',
@@ -1 +1 @@
1
- {"version":3,"file":"addJobScheduler-11.js","sourceRoot":"","sources":["../../../src/scripts/addJobScheduler-11.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAucf,CAAC;AACF,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,IAAI,EAAE,iBAAiB;IACvB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
1
+ {"version":3,"file":"addJobScheduler-11.js","sourceRoot":"","sources":["../../../src/scripts/addJobScheduler-11.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0hBf,CAAC;AACF,MAAM,CAAC,MAAM,eAAe,GAAG;IAC7B,IAAI,EAAE,iBAAiB;IACvB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
@@ -7,7 +7,7 @@ const content = `--[[
7
7
  ARGV[2] msgpacked options
8
8
  [1] name
9
9
  [2] tz?
10
- [3] patten?
10
+ [3] pattern?
11
11
  [4] endDate?
12
12
  [5] every?
13
13
  ARGV[3] legacy custom key TODO: remove this logic in next breaking change
@@ -0,0 +1,5 @@
1
+ export declare const getMetrics: {
2
+ name: string;
3
+ content: string;
4
+ keys: number;
5
+ };
@@ -0,0 +1,22 @@
1
+ const content = `--[[
2
+ Get metrics
3
+ Input:
4
+ KEYS[1] 'metrics' key
5
+ KEYS[2] 'metrics data' key
6
+ ARGV[1] start index
7
+ ARGV[2] end index
8
+ ]]
9
+ local rcall = redis.call;
10
+ local metricsKey = KEYS[1]
11
+ local dataKey = KEYS[2]
12
+ local metrics = rcall("HMGET", metricsKey, "count", "prevTS", "prevCount")
13
+ local data = rcall("LRANGE", dataKey, tonumber(ARGV[1]), tonumber(ARGV[2]))
14
+ local numPoints = rcall("LLEN", dataKey)
15
+ return {metrics, data, numPoints}
16
+ `;
17
+ export const getMetrics = {
18
+ name: 'getMetrics',
19
+ content,
20
+ keys: 2,
21
+ };
22
+ //# sourceMappingURL=getMetrics-2.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"getMetrics-2.js","sourceRoot":"","sources":["../../../src/scripts/getMetrics-2.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;CAef,CAAC;AACF,MAAM,CAAC,MAAM,UAAU,GAAG;IACxB,IAAI,EAAE,YAAY;IAClB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -15,6 +15,7 @@ export * from './getCounts-1';
15
15
  export * from './getCountsPerPriority-4';
16
16
  export * from './getDependencyCounts-4';
17
17
  export * from './getJobScheduler-1';
18
+ export * from './getMetrics-2';
18
19
  export * from './getRanges-1';
19
20
  export * from './getRateLimitTtl-1';
20
21
  export * from './getState-8';
@@ -15,6 +15,7 @@ export * from './getCounts-1';
15
15
  export * from './getCountsPerPriority-4';
16
16
  export * from './getDependencyCounts-4';
17
17
  export * from './getJobScheduler-1';
18
+ export * from './getMetrics-2';
18
19
  export * from './getRanges-1';
19
20
  export * from './getRateLimitTtl-1';
20
21
  export * from './getState-8';
@@ -1 +1 @@
1
- {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/scripts/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,sBAAsB,CAAC;AACrC,cAAc,YAAY,CAAC;AAC3B,cAAc,kBAAkB,CAAC;AACjC,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AACnC,cAAc,oBAAoB,CAAC;AACnC,cAAc,WAAW,CAAC;AAC1B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,eAAe,CAAC;AAC9B,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC;AACpC,cAAc,eAAe,CAAC;AAC9B,cAAc,qBAAqB,CAAC;AACpC,cAAc,cAAc,CAAC;AAC7B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,aAAa,CAAC;AAC5B,cAAc,6BAA6B,CAAC;AAC5C,cAAc,oBAAoB,CAAC;AACnC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,mBAAmB,CAAC;AAClC,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,gBAAgB,CAAC;AAC/B,cAAc,cAAc,CAAC;AAC7B,cAAc,WAAW,CAAC;AAC1B,cAAc,aAAa,CAAC;AAC5B,cAAc,iBAAiB,CAAC;AAChC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,eAAe,CAAC;AAC9B,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,kBAAkB,CAAC;AACjC,cAAc,eAAe,CAAC;AAC9B,cAAc,oBAAoB,CAAC;AACnC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,yBAAyB,CAAC;AACxC,cAAc,oBAAoB,CAAC;AACnC,cAAc,+BAA+B,CAAC"}
1
+ {"version":3,"file":"index.js","sourceRoot":"","sources":["../../../src/scripts/index.ts"],"names":[],"mappings":"AAAA,cAAc,mBAAmB,CAAC;AAClC,cAAc,sBAAsB,CAAC;AACrC,cAAc,YAAY,CAAC;AAC3B,cAAc,kBAAkB,CAAC;AACjC,cAAc,uBAAuB,CAAC;AACtC,cAAc,sBAAsB,CAAC;AACrC,cAAc,oBAAoB,CAAC;AACnC,cAAc,iBAAiB,CAAC;AAChC,cAAc,oBAAoB,CAAC;AACnC,cAAc,oBAAoB,CAAC;AACnC,cAAc,WAAW,CAAC;AAC1B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,eAAe,CAAC;AAC9B,cAAc,0BAA0B,CAAC;AACzC,cAAc,yBAAyB,CAAC;AACxC,cAAc,qBAAqB,CAAC;AACpC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,eAAe,CAAC;AAC9B,cAAc,qBAAqB,CAAC;AACpC,cAAc,cAAc,CAAC;AAC7B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,gBAAgB,CAAC;AAC/B,cAAc,iBAAiB,CAAC;AAChC,cAAc,aAAa,CAAC;AAC5B,cAAc,6BAA6B,CAAC;AAC5C,cAAc,oBAAoB,CAAC;AACnC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,mBAAmB,CAAC;AAClC,cAAc,mBAAmB,CAAC;AAClC,cAAc,qBAAqB,CAAC;AACpC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,gBAAgB,CAAC;AAC/B,cAAc,cAAc,CAAC;AAC7B,cAAc,WAAW,CAAC;AAC1B,cAAc,aAAa,CAAC;AAC5B,cAAc,iBAAiB,CAAC;AAChC,cAAc,2BAA2B,CAAC;AAC1C,cAAc,eAAe,CAAC;AAC9B,cAAc,wBAAwB,CAAC;AACvC,cAAc,sBAAsB,CAAC;AACrC,cAAc,+BAA+B,CAAC;AAC9C,cAAc,kBAAkB,CAAC;AACjC,cAAc,eAAe,CAAC;AAC9B,cAAc,oBAAoB,CAAC;AACnC,cAAc,gBAAgB,CAAC;AAC/B,cAAc,yBAAyB,CAAC;AACxC,cAAc,oBAAoB,CAAC;AACnC,cAAc,+BAA+B,CAAC"}
@@ -141,7 +141,17 @@ if (#stalling > 0) then
141
141
  if (removed > 0) then
142
142
  -- If this job has been stalled too many times, such as if it crashes the worker, then fail it.
143
143
  local stalledCount = rcall("HINCRBY", jobKey, "stc", 1)
144
- if stalledCount > maxStalledJobCount then
144
+ -- Check if this is a repeatable job by looking at job options
145
+ local jobOpts = rcall("HGET", jobKey, "opts")
146
+ local isRepeatableJob = false
147
+ if jobOpts then
148
+ local opts = cjson.decode(jobOpts)
149
+ if opts and opts["repeat"] then
150
+ isRepeatableJob = true
151
+ end
152
+ end
153
+ -- Only fail job if it exceeds stall limit AND is not a repeatable job
154
+ if stalledCount > maxStalledJobCount and not isRepeatableJob then
145
155
  local failedReason = "job stalled more than allowable limit"
146
156
  rcall("HSET", jobKey, "defa", failedReason)
147
157
  end
@@ -1 +1 @@
1
- {"version":3,"file":"moveStalledJobsToWait-8.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqKf,CAAC;AACF,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"moveStalledJobsToWait-8.js","sourceRoot":"","sources":["../../../src/scripts/moveStalledJobsToWait-8.ts"],"names":[],"mappings":"AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA+Kf,CAAC;AACF,MAAM,CAAC,MAAM,qBAAqB,GAAG;IACnC,IAAI,EAAE,uBAAuB;IAC7B,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}