bullmq 2.4.0 → 3.0.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (122) hide show
  1. package/README.md +1 -1
  2. package/dist/cjs/classes/backoffs.d.ts +3 -6
  3. package/dist/cjs/classes/backoffs.js +8 -8
  4. package/dist/cjs/classes/backoffs.js.map +1 -1
  5. package/dist/cjs/classes/flow-producer.js +2 -1
  6. package/dist/cjs/classes/flow-producer.js.map +1 -1
  7. package/dist/cjs/classes/job.d.ts +3 -3
  8. package/dist/cjs/classes/job.js +1 -1
  9. package/dist/cjs/classes/job.js.map +1 -1
  10. package/dist/cjs/classes/queue-base.js +1 -1
  11. package/dist/cjs/classes/queue-base.js.map +1 -1
  12. package/dist/cjs/classes/queue.d.ts +8 -6
  13. package/dist/cjs/classes/queue.js +10 -9
  14. package/dist/cjs/classes/queue.js.map +1 -1
  15. package/dist/cjs/classes/repeat.d.ts +0 -1
  16. package/dist/cjs/classes/repeat.js +4 -5
  17. package/dist/cjs/classes/repeat.js.map +1 -1
  18. package/dist/cjs/classes/scripts.d.ts +6 -6
  19. package/dist/cjs/classes/scripts.js +43 -22
  20. package/dist/cjs/classes/scripts.js.map +1 -1
  21. package/dist/cjs/classes/worker.d.ts +12 -3
  22. package/dist/cjs/classes/worker.js +50 -34
  23. package/dist/cjs/classes/worker.js.map +1 -1
  24. package/dist/cjs/commands/includes/getRateLimitTTL.lua +13 -0
  25. package/dist/cjs/commands/includes/moveJobFromWaitToActive.lua +15 -34
  26. package/dist/cjs/commands/includes/promoteDelayedJobs.lua +0 -6
  27. package/dist/cjs/commands/moveToActive-9.lua +11 -2
  28. package/dist/cjs/commands/moveToDelayed-8.lua +63 -0
  29. package/dist/cjs/commands/moveToFinished-12.lua +8 -1
  30. package/dist/{esm/commands/retryJob-6.lua → cjs/commands/retryJob-8.lua} +15 -8
  31. package/dist/cjs/interfaces/advanced-options.d.ts +4 -4
  32. package/dist/cjs/interfaces/base-job-options.d.ts +0 -6
  33. package/dist/cjs/interfaces/queue-options.d.ts +0 -10
  34. package/dist/cjs/interfaces/rate-limiter-options.d.ts +0 -18
  35. package/dist/cjs/interfaces/repeat-options.d.ts +1 -6
  36. package/dist/cjs/scripts/index.d.ts +2 -2
  37. package/dist/cjs/scripts/index.js +2 -2
  38. package/dist/cjs/scripts/moveToActive-9.js +35 -36
  39. package/dist/cjs/scripts/moveToActive-9.js.map +1 -1
  40. package/dist/cjs/scripts/{moveToDelayed-5.d.ts → moveToDelayed-8.d.ts} +0 -0
  41. package/dist/cjs/scripts/moveToDelayed-8.js +116 -0
  42. package/dist/cjs/scripts/moveToDelayed-8.js.map +1 -0
  43. package/dist/cjs/scripts/moveToFinished-12.js +33 -36
  44. package/dist/cjs/scripts/moveToFinished-12.js.map +1 -1
  45. package/dist/cjs/scripts/{retryJob-6.d.ts → retryJob-8.d.ts} +0 -0
  46. package/dist/cjs/scripts/retryJob-8.js +112 -0
  47. package/dist/cjs/scripts/retryJob-8.js.map +1 -0
  48. package/dist/cjs/types/backoff-strategy.d.ts +2 -0
  49. package/dist/cjs/types/backoff-strategy.js +3 -0
  50. package/dist/cjs/types/backoff-strategy.js.map +1 -0
  51. package/dist/cjs/types/index.d.ts +2 -1
  52. package/dist/cjs/types/index.js +2 -1
  53. package/dist/cjs/types/index.js.map +1 -1
  54. package/dist/cjs/utils.d.ts +1 -3
  55. package/dist/cjs/utils.js +1 -13
  56. package/dist/cjs/utils.js.map +1 -1
  57. package/dist/esm/classes/backoffs.d.ts +3 -6
  58. package/dist/esm/classes/backoffs.js +8 -8
  59. package/dist/esm/classes/backoffs.js.map +1 -1
  60. package/dist/esm/classes/flow-producer.js +3 -2
  61. package/dist/esm/classes/flow-producer.js.map +1 -1
  62. package/dist/esm/classes/job.d.ts +3 -3
  63. package/dist/esm/classes/job.js +1 -1
  64. package/dist/esm/classes/job.js.map +1 -1
  65. package/dist/esm/classes/queue-base.js +1 -1
  66. package/dist/esm/classes/queue-base.js.map +1 -1
  67. package/dist/esm/classes/queue.d.ts +8 -6
  68. package/dist/esm/classes/queue.js +11 -10
  69. package/dist/esm/classes/queue.js.map +1 -1
  70. package/dist/esm/classes/repeat.d.ts +0 -1
  71. package/dist/esm/classes/repeat.js +4 -5
  72. package/dist/esm/classes/repeat.js.map +1 -1
  73. package/dist/esm/classes/scripts.d.ts +6 -6
  74. package/dist/esm/classes/scripts.js +41 -20
  75. package/dist/esm/classes/scripts.js.map +1 -1
  76. package/dist/esm/classes/worker.d.ts +12 -3
  77. package/dist/esm/classes/worker.js +50 -34
  78. package/dist/esm/classes/worker.js.map +1 -1
  79. package/dist/esm/commands/includes/getRateLimitTTL.lua +13 -0
  80. package/dist/esm/commands/includes/moveJobFromWaitToActive.lua +15 -34
  81. package/dist/esm/commands/includes/promoteDelayedJobs.lua +0 -6
  82. package/dist/esm/commands/moveToActive-9.lua +11 -2
  83. package/dist/esm/commands/moveToDelayed-8.lua +63 -0
  84. package/dist/esm/commands/moveToFinished-12.lua +8 -1
  85. package/dist/{cjs/commands/retryJob-6.lua → esm/commands/retryJob-8.lua} +15 -8
  86. package/dist/esm/interfaces/advanced-options.d.ts +4 -4
  87. package/dist/esm/interfaces/base-job-options.d.ts +0 -6
  88. package/dist/esm/interfaces/queue-options.d.ts +0 -10
  89. package/dist/esm/interfaces/rate-limiter-options.d.ts +0 -18
  90. package/dist/esm/interfaces/repeat-options.d.ts +1 -6
  91. package/dist/esm/scripts/index.d.ts +2 -2
  92. package/dist/esm/scripts/index.js +2 -2
  93. package/dist/esm/scripts/moveToActive-9.js +35 -36
  94. package/dist/esm/scripts/moveToActive-9.js.map +1 -1
  95. package/dist/esm/scripts/{moveToDelayed-5.d.ts → moveToDelayed-8.d.ts} +0 -0
  96. package/dist/esm/scripts/moveToDelayed-8.js +113 -0
  97. package/dist/esm/scripts/moveToDelayed-8.js.map +1 -0
  98. package/dist/esm/scripts/moveToFinished-12.js +33 -36
  99. package/dist/esm/scripts/moveToFinished-12.js.map +1 -1
  100. package/dist/esm/scripts/{retryJob-6.d.ts → retryJob-8.d.ts} +0 -0
  101. package/dist/esm/scripts/retryJob-8.js +109 -0
  102. package/dist/esm/scripts/retryJob-8.js.map +1 -0
  103. package/dist/esm/types/backoff-strategy.d.ts +2 -0
  104. package/dist/esm/types/backoff-strategy.js +2 -0
  105. package/dist/esm/types/backoff-strategy.js.map +1 -0
  106. package/dist/esm/types/index.d.ts +2 -1
  107. package/dist/esm/types/index.js +2 -1
  108. package/dist/esm/types/index.js.map +1 -1
  109. package/dist/esm/utils.d.ts +1 -3
  110. package/dist/esm/utils.js +0 -11
  111. package/dist/esm/utils.js.map +1 -1
  112. package/package.json +1 -1
  113. package/dist/cjs/commands/moveToDelayed-5.lua +0 -54
  114. package/dist/cjs/scripts/moveToDelayed-5.js +0 -52
  115. package/dist/cjs/scripts/moveToDelayed-5.js.map +0 -1
  116. package/dist/cjs/scripts/retryJob-6.js +0 -60
  117. package/dist/cjs/scripts/retryJob-6.js.map +0 -1
  118. package/dist/esm/commands/moveToDelayed-5.lua +0 -54
  119. package/dist/esm/scripts/moveToDelayed-5.js +0 -49
  120. package/dist/esm/scripts/moveToDelayed-5.js.map +0 -1
  121. package/dist/esm/scripts/retryJob-6.js +0 -57
  122. package/dist/esm/scripts/retryJob-6.js.map +0 -1
@@ -0,0 +1,63 @@
1
+ --[[
2
+ Moves job from active to delayed set.
3
+
4
+ Input:
5
+ KEYS[1] wait key
6
+ KEYS[2] active key
7
+ KEYS[3] priority key
8
+ KEYS[4] delayed key
9
+ KEYS[5] job key
10
+ KEYS[6] events stream
11
+ KEYS[7] paused key
12
+ KEYS[8] meta key
13
+
14
+ ARGV[1] key prefix
15
+ ARGV[2] timestamp
16
+ ARGV[3] delayedTimestamp
17
+ ARGV[4] the id of the job
18
+ ARGV[5] queue token
19
+
20
+ Output:
21
+ 0 - OK
22
+ -1 - Missing job.
23
+ -3 - Job not in active set.
24
+
25
+ Events:
26
+ - delayed key.
27
+ ]]
28
+ local rcall = redis.call
29
+
30
+ -- Includes
31
+ --- @include "includes/promoteDelayedJobs"
32
+
33
+ promoteDelayedJobs(KEYS[4], KEYS[1], KEYS[3], KEYS[7], KEYS[8], KEYS[6], ARGV[1], ARGV[2])
34
+
35
+ if rcall("EXISTS", KEYS[5]) == 1 then
36
+
37
+ if ARGV[5] ~= "0" then
38
+ local lockKey = KEYS[5] .. ':lock'
39
+ if rcall("GET", lockKey) == ARGV[5] then
40
+ rcall("DEL", lockKey)
41
+ else
42
+ return -2
43
+ end
44
+ end
45
+
46
+ local jobId = ARGV[4]
47
+ local score = tonumber(ARGV[3])
48
+ local delayedTimestamp = (score / 0x1000)
49
+
50
+ local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
51
+
52
+ if(numRemovedElements < 1) then
53
+ return -3
54
+ end
55
+
56
+ rcall("ZADD", KEYS[4], score, jobId)
57
+
58
+ rcall("XADD", KEYS[6], "*", "event", "delayed", "jobId", jobId, "delay", delayedTimestamp);
59
+
60
+ return 0
61
+ else
62
+ return -1
63
+ end
@@ -64,6 +64,7 @@ local rcall = redis.call
64
64
  --- @include "includes/removeJobsByMaxCount"
65
65
  --- @include "includes/trimEvents"
66
66
  --- @include "includes/updateParentDepsIfNeeded"
67
+ --- @include "includes/getRateLimitTTL"
67
68
 
68
69
  local jobIdKey = KEYS[10]
69
70
  if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
@@ -169,6 +170,12 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
169
170
  -- Check if there are delayed jobs that can be promoted
170
171
  promoteDelayedJobs(KEYS[7], KEYS[1], KEYS[3], KEYS[8], KEYS[11], KEYS[4], ARGV[8], timestamp)
171
172
 
173
+ -- Check if we are rate limited first.
174
+ local pttl = getRateLimitTTL(opts, KEYS[6])
175
+ if pttl > 0 then
176
+ return { 0, 0, pttl }
177
+ end
178
+
172
179
  jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
173
180
 
174
181
  if jobId == "0" then
@@ -182,7 +189,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
182
189
  if (nextTimestamp ~= nil) then
183
190
  -- The result is guaranteed to be positive, since the
184
191
  -- ZRANGEBYSCORE command would have return a job otherwise.
185
- return nextTimestamp - timestamp
192
+ return {0, 0, 0, nextTimestamp - timestamp}
186
193
  end
187
194
  end
188
195
 
@@ -8,10 +8,14 @@
8
8
  KEYS[4] job key
9
9
  KEYS[5] 'meta'
10
10
  KEYS[6] events stream
11
+ KEYS[7] delayed key
12
+ KEYS[8] priority key
11
13
 
12
- ARGV[1] pushCmd
13
- ARGV[2] jobId
14
- ARGV[3] token
14
+ ARGV[1] key prefix
15
+ ARGV[2] timestamp
16
+ ARGV[3] pushCmd
17
+ ARGV[4] jobId
18
+ ARGV[5] token
15
19
 
16
20
  Events:
17
21
  'waiting'
@@ -25,12 +29,15 @@ local rcall = redis.call
25
29
 
26
30
  -- Includes
27
31
  --- @include "includes/getTargetQueueList"
32
+ --- @include "includes/promoteDelayedJobs"
33
+
34
+ promoteDelayedJobs(KEYS[7], KEYS[2], KEYS[8], KEYS[3], KEYS[5], KEYS[6], ARGV[1], ARGV[2])
28
35
 
29
36
  if rcall("EXISTS", KEYS[4]) == 1 then
30
37
 
31
- if ARGV[3] ~= "0" then
38
+ if ARGV[5] ~= "0" then
32
39
  local lockKey = KEYS[4] .. ':lock'
33
- if rcall("GET", lockKey) == ARGV[3] then
40
+ if rcall("GET", lockKey) == ARGV[5] then
34
41
  rcall("DEL", lockKey)
35
42
  else
36
43
  return -2
@@ -39,11 +46,11 @@ if rcall("EXISTS", KEYS[4]) == 1 then
39
46
 
40
47
  local target = getTargetQueueList(KEYS[5], KEYS[2], KEYS[3])
41
48
 
42
- rcall("LREM", KEYS[1], 0, ARGV[2])
43
- rcall(ARGV[1], target, ARGV[2])
49
+ rcall("LREM", KEYS[1], 0, ARGV[4])
50
+ rcall(ARGV[3], target, ARGV[4])
44
51
 
45
52
  -- Emit waiting event
46
- rcall("XADD", KEYS[6], "*", "event", "waiting", "jobId", ARGV[2], "prev", "failed");
53
+ rcall("XADD", KEYS[6], "*", "event", "waiting", "jobId", ARGV[4], "prev", "failed");
47
54
 
48
55
  return 0
49
56
  else
@@ -1,13 +1,13 @@
1
- import { RepeatStrategy } from '../types';
1
+ import { BackoffStrategy, RepeatStrategy } from '../types';
2
2
  export interface AdvancedRepeatOptions {
3
3
  /**
4
- * A set of custom cron strategies keyed by name.
4
+ * A custom cron strategy.
5
5
  */
6
6
  repeatStrategy?: RepeatStrategy;
7
7
  }
8
8
  export interface AdvancedOptions extends AdvancedRepeatOptions {
9
9
  /**
10
- * A set of custom backoff strategies keyed by name.
10
+ * A custom backoff strategy.
11
11
  */
12
- backoffStrategies?: {};
12
+ backoffStrategy?: BackoffStrategy;
13
13
  }
@@ -23,12 +23,6 @@ export interface DefaultJobOptions {
23
23
  * @defaultValue 0
24
24
  */
25
25
  attempts?: number;
26
- /**
27
- * Rate limiter key to use if rate limiter enabled.
28
- *
29
- * @see {@link https://docs.bullmq.io/guide/rate-limiting}
30
- */
31
- rateLimiterKey?: string;
32
26
  /**
33
27
  * Backoff setting for automatic retries if the job fails
34
28
  */
@@ -31,16 +31,6 @@ export interface QueueBaseOptions {
31
31
  */
32
32
  export interface QueueOptions extends QueueBaseOptions {
33
33
  defaultJobOptions?: DefaultJobOptions;
34
- /**
35
- * Options for the rate limiter.
36
- */
37
- limiter?: {
38
- /**
39
- * Group key to be used by the limiter when
40
- * limiting by group keys.
41
- */
42
- groupKey: string;
43
- };
44
34
  /**
45
35
  * Options for the streams used internally in BullMQ.
46
36
  */
@@ -9,22 +9,4 @@ export interface RateLimiterOptions {
9
9
  * of `max` jobs will be processed.
10
10
  */
11
11
  duration: number;
12
- /**
13
- * It is possible to define a rate limiter based on group keys,
14
- * for example you may want to have a rate limiter per customer
15
- * instead of a global rate limiter for all customers
16
- *
17
- * @see {@link https://docs.bullmq.io/guide/rate-limiting}
18
- */
19
- groupKey?: string;
20
- /**
21
- * This option enables a heuristic so that when a queue is heavily
22
- * rete limited, it delays the workers so that they do not try
23
- * to pick jobs when there is no point in doing so.
24
- * Note: It is not recommended to use this option when using
25
- * groupKeys unless you have a big amount of workers since
26
- * you may be delaying workers that could pick jobs in groups that
27
- * have not been rate limited.
28
- */
29
- workerDelay?: boolean;
30
12
  }
@@ -5,11 +5,6 @@ import { ParserOptions } from 'cron-parser';
5
5
  * @see {@link https://docs.bullmq.io/guide/jobs/repeatable}
6
6
  */
7
7
  export interface RepeatOptions extends Omit<ParserOptions, 'iterator'> {
8
- /**
9
- * @deprecated Use pattern option instead.
10
- * A cron pattern
11
- */
12
- cron?: string;
13
8
  /**
14
9
  * A repeat pattern
15
10
  */
@@ -20,7 +15,7 @@ export interface RepeatOptions extends Omit<ParserOptions, 'iterator'> {
20
15
  limit?: number;
21
16
  /**
22
17
  * Repeat after this amount of milliseconds
23
- * (`cron` setting cannot be used together with this setting.)
18
+ * (`pattern` setting cannot be used together with this setting.)
24
19
  */
25
20
  every?: number;
26
21
  /**
@@ -9,7 +9,7 @@ export * from './isFinished-3';
9
9
  export * from './isJobInList-1';
10
10
  export * from './moveStalledJobsToWait-8';
11
11
  export * from './moveToActive-9';
12
- export * from './moveToDelayed-5';
12
+ export * from './moveToDelayed-8';
13
13
  export * from './moveToFinished-12';
14
14
  export * from './moveToWaitingChildren-4';
15
15
  export * from './obliterate-2';
@@ -19,7 +19,7 @@ export * from './releaseLock-1';
19
19
  export * from './removeJob-1';
20
20
  export * from './removeRepeatable-2';
21
21
  export * from './reprocessJob-4';
22
- export * from './retryJob-6';
22
+ export * from './retryJob-8';
23
23
  export * from './retryJobs-6';
24
24
  export * from './takeLock-1';
25
25
  export * from './updateData-1';
@@ -12,7 +12,7 @@ tslib_1.__exportStar(require("./isFinished-3"), exports);
12
12
  tslib_1.__exportStar(require("./isJobInList-1"), exports);
13
13
  tslib_1.__exportStar(require("./moveStalledJobsToWait-8"), exports);
14
14
  tslib_1.__exportStar(require("./moveToActive-9"), exports);
15
- tslib_1.__exportStar(require("./moveToDelayed-5"), exports);
15
+ tslib_1.__exportStar(require("./moveToDelayed-8"), exports);
16
16
  tslib_1.__exportStar(require("./moveToFinished-12"), exports);
17
17
  tslib_1.__exportStar(require("./moveToWaitingChildren-4"), exports);
18
18
  tslib_1.__exportStar(require("./obliterate-2"), exports);
@@ -22,7 +22,7 @@ tslib_1.__exportStar(require("./releaseLock-1"), exports);
22
22
  tslib_1.__exportStar(require("./removeJob-1"), exports);
23
23
  tslib_1.__exportStar(require("./removeRepeatable-2"), exports);
24
24
  tslib_1.__exportStar(require("./reprocessJob-4"), exports);
25
- tslib_1.__exportStar(require("./retryJob-6"), exports);
25
+ tslib_1.__exportStar(require("./retryJob-8"), exports);
26
26
  tslib_1.__exportStar(require("./retryJobs-6"), exports);
27
27
  tslib_1.__exportStar(require("./takeLock-1"), exports);
28
28
  tslib_1.__exportStar(require("./updateData-1"), exports);
@@ -49,41 +49,25 @@ local rcall = redis.call
49
49
  local function moveJobFromWaitToActive(keys, keyPrefix, jobId, processedOn, opts)
50
50
  -- Check if we need to perform rate limiting.
51
51
  local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
52
+ local expireTime
52
53
  if(maxJobs) then
53
54
  local rateLimiterKey = keys[6];
54
- local groupKey
55
- local groupKeyOpt = opts['limiter'] and opts['limiter']['groupKey'] or ""
56
- if groupKeyOpt ~= "" then
57
- groupKey = string.match(jobId, "[^:]+$")
58
- if groupKey ~= jobId then
59
- rateLimiterKey = rateLimiterKey .. ":" .. groupKey
60
- end
55
+ local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
56
+ if jobCounter == 1 then
57
+ local limiterDuration = opts['limiter'] and opts['limiter']['duration']
58
+ rcall("PEXPIRE", rateLimiterKey, limiterDuration)
61
59
  end
62
- local jobCounter
63
- if groupKey ~= nil then
64
- if rateLimiterKey ~= keys[6] then
65
- jobCounter = tonumber(rcall("INCR", rateLimiterKey))
66
- end
67
- else
68
- jobCounter = tonumber(rcall("INCR", rateLimiterKey))
69
- end
70
- local limiterDuration = opts['limiter'] and opts['limiter']['duration']
71
- -- check if rate limit hit
72
- if jobCounter ~= nil and jobCounter > maxJobs then
73
- local exceedingJobs = jobCounter - maxJobs
74
- local expireTime = tonumber(rcall("PTTL", rateLimiterKey))
75
- local delay = expireTime + ((exceedingJobs - 1) * limiterDuration) / maxJobs;
76
- local timestamp = delay + tonumber(processedOn)
77
- -- put job into delayed queue
78
- rcall("ZADD", keys[7], timestamp * 0x1000 + bit.band(jobCounter, 0xfff), jobId);
79
- rcall("XADD", keys[4], "*", "event", "delayed", "jobId", jobId, "delay", timestamp);
80
- -- remove from active queue
60
+ -- check if we passed rate limit, we need to remove the job and return expireTime
61
+ if jobCounter > maxJobs then
62
+ expireTime = rcall("PTTL", rateLimiterKey)
63
+ -- remove from active queue and add back to the wait list
81
64
  rcall("LREM", keys[2], 1, jobId)
65
+ rcall("RPUSH", keys[1], jobId)
82
66
  -- Return when we can process more jobs
83
- return expireTime
67
+ return {0, 0, expireTime}
84
68
  else
85
- if jobCounter == 1 then
86
- rcall("PEXPIRE", rateLimiterKey, limiterDuration)
69
+ if jobCounter == maxJobs then
70
+ expireTime = rcall("PTTL", rateLimiterKey)
87
71
  end
88
72
  end
89
73
  end
@@ -97,7 +81,7 @@ local function moveJobFromWaitToActive(keys, keyPrefix, jobId, processedOn, opts
97
81
  rcall("XADD", keys[4], "*", "event", "active", "jobId", jobId, "prev", "waiting")
98
82
  rcall("HSET", jobKey, "processedOn", processedOn)
99
83
  rcall("HINCRBY", jobKey, "attemptsMade", 1)
100
- return {rcall("HGETALL", jobKey), jobId} -- get job data
84
+ return {rcall("HGETALL", jobKey), jobId, expireTime} -- get job data
101
85
  end
102
86
  --[[
103
87
  Function to return the next delayed job timestamp.
@@ -169,19 +153,34 @@ local function promoteDelayedJobs(delayedKey, waitKey, priorityKey, pausedKey,
169
153
  rcall("HSET", prefix .. jobId, "delay", 0)
170
154
  end
171
155
  end
172
- local nextTimestamp = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")[2]
173
- if (nextTimestamp ~= nil) then
174
- nextTimestamp = nextTimestamp / 0x1000
156
+ end
157
+ local function getRateLimitTTL(opts, limiterKey)
158
+ local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
159
+ if maxJobs then
160
+ local jobCounter = tonumber(rcall("GET", limiterKey))
161
+ if jobCounter ~= nil and jobCounter >= maxJobs then
162
+ local pttl = rcall("PTTL", KEYS[6])
163
+ if pttl > 0 then
164
+ return pttl
165
+ end
166
+ end
175
167
  end
176
- return nextTimestamp
168
+ return 0
177
169
  end
178
170
  -- Check if there are delayed jobs that we can move to wait.
179
171
  promoteDelayedJobs(KEYS[7], KEYS[1], KEYS[3], KEYS[8], KEYS[9], KEYS[4], ARGV[1], ARGV[2])
172
+ local opts
180
173
  if (ARGV[3] ~= "") then
181
174
  jobId = ARGV[3]
182
175
  -- clean stalled key
183
176
  rcall("SREM", KEYS[5], jobId)
184
177
  else
178
+ -- Check if we are rate limited first.
179
+ opts = cmsgpack.unpack(ARGV[4])
180
+ local pttl = getRateLimitTTL(opts, KEYS[6])
181
+ if pttl > 0 then
182
+ return { 0, 0, pttl }
183
+ end
185
184
  -- no job ID, try non-blocking move from wait to active
186
185
  jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
187
186
  end
@@ -189,14 +188,14 @@ end
189
188
  if jobId == "0" then
190
189
  rcall("LREM", KEYS[2], 1, 0)
191
190
  elseif jobId then
192
- local opts = cmsgpack.unpack(ARGV[4])
191
+ opts = opts or cmsgpack.unpack(ARGV[4])
193
192
  -- this script is not really moving, it is preparing the job for processing
194
193
  return moveJobFromWaitToActive(KEYS, ARGV[1], jobId, ARGV[2], opts)
195
194
  end
196
195
  -- Return the timestamp for the next delayed job if any.
197
196
  local nextTimestamp = getNextDelayedTimestamp(KEYS[7])
198
197
  if (nextTimestamp ~= nil) then
199
- return nextTimestamp - tonumber(ARGV[2])
198
+ return { 0, 0, 0, nextTimestamp - tonumber(ARGV[2])}
200
199
  end
201
200
  `;
202
201
  exports.moveToActive = {
@@ -1 +1 @@
1
- {"version":3,"file":"moveToActive-9.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqMf,CAAC;AACW,QAAA,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
1
+ {"version":3,"file":"moveToActive-9.js","sourceRoot":"","sources":["../../../src/scripts/moveToActive-9.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAoMf,CAAC;AACW,QAAA,YAAY,GAAG;IAC1B,IAAI,EAAE,cAAc;IACpB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -0,0 +1,116 @@
1
+ "use strict";
2
+ Object.defineProperty(exports, "__esModule", { value: true });
3
+ exports.moveToDelayed = void 0;
4
+ const content = `--[[
5
+ Moves job from active to delayed set.
6
+ Input:
7
+ KEYS[1] wait key
8
+ KEYS[2] active key
9
+ KEYS[3] priority key
10
+ KEYS[4] delayed key
11
+ KEYS[5] job key
12
+ KEYS[6] events stream
13
+ KEYS[7] paused key
14
+ KEYS[8] meta key
15
+ ARGV[1] key prefix
16
+ ARGV[2] timestamp
17
+ ARGV[3] delayedTimestamp
18
+ ARGV[4] the id of the job
19
+ ARGV[5] queue token
20
+ Output:
21
+ 0 - OK
22
+ -1 - Missing job.
23
+ -3 - Job not in active set.
24
+ Events:
25
+ - delayed key.
26
+ ]]
27
+ local rcall = redis.call
28
+ -- Includes
29
+ --[[
30
+ Updates the delay set, by moving delayed jobs that should
31
+ be processed now to "wait".
32
+ Events:
33
+ 'waiting'
34
+ ]]
35
+ local rcall = redis.call
36
+ -- Includes
37
+ --[[
38
+ Function to add job considering priority.
39
+ ]]
40
+ local function addJobWithPriority(priorityKey, priority, targetKey, jobId)
41
+ rcall("ZADD", priorityKey, priority, jobId)
42
+ local count = rcall("ZCOUNT", priorityKey, 0, priority)
43
+ local len = rcall("LLEN", targetKey)
44
+ local id = rcall("LINDEX", targetKey, len - (count - 1))
45
+ if id then
46
+ rcall("LINSERT", targetKey, "BEFORE", id, jobId)
47
+ else
48
+ rcall("RPUSH", targetKey, jobId)
49
+ end
50
+ end
51
+ --[[
52
+ Function to check for the meta.paused key to decide if we are paused or not
53
+ (since an empty list and !EXISTS are not really the same).
54
+ ]]
55
+ local function getTargetQueueList(queueMetaKey, waitKey, pausedKey)
56
+ if rcall("HEXISTS", queueMetaKey, "paused") ~= 1 then
57
+ return waitKey
58
+ else
59
+ return pausedKey
60
+ end
61
+ end
62
+ -- Try to get as much as 1000 jobs at once, and returns the nextTimestamp if
63
+ -- there are more delayed jobs to process.
64
+ local function promoteDelayedJobs(delayedKey, waitKey, priorityKey, pausedKey,
65
+ metaKey, eventStreamKey, prefix, timestamp)
66
+ local jobs = rcall("ZRANGEBYSCORE", delayedKey, 0, (timestamp + 1) * 0x1000, "LIMIT", 0, 1000)
67
+ if (#jobs > 0) then
68
+ rcall("ZREM", delayedKey, unpack(jobs))
69
+ -- check if we need to use push in paused instead of waiting
70
+ local target = getTargetQueueList(metaKey, waitKey, pausedKey)
71
+ for _, jobId in ipairs(jobs) do
72
+ local priority =
73
+ tonumber(rcall("HGET", prefix .. jobId, "priority")) or 0
74
+ if priority == 0 then
75
+ -- LIFO or FIFO
76
+ rcall("LPUSH", target, jobId)
77
+ else
78
+ addJobWithPriority(priorityKey, priority, target, jobId)
79
+ end
80
+ -- Emit waiting event
81
+ rcall("XADD", eventStreamKey, "*", "event", "waiting", "jobId",
82
+ jobId, "prev", "delayed")
83
+ rcall("HSET", prefix .. jobId, "delay", 0)
84
+ end
85
+ end
86
+ end
87
+ promoteDelayedJobs(KEYS[4], KEYS[1], KEYS[3], KEYS[7], KEYS[8], KEYS[6], ARGV[1], ARGV[2])
88
+ if rcall("EXISTS", KEYS[5]) == 1 then
89
+ if ARGV[5] ~= "0" then
90
+ local lockKey = KEYS[5] .. ':lock'
91
+ if rcall("GET", lockKey) == ARGV[5] then
92
+ rcall("DEL", lockKey)
93
+ else
94
+ return -2
95
+ end
96
+ end
97
+ local jobId = ARGV[4]
98
+ local score = tonumber(ARGV[3])
99
+ local delayedTimestamp = (score / 0x1000)
100
+ local numRemovedElements = rcall("LREM", KEYS[2], -1, jobId)
101
+ if(numRemovedElements < 1) then
102
+ return -3
103
+ end
104
+ rcall("ZADD", KEYS[4], score, jobId)
105
+ rcall("XADD", KEYS[6], "*", "event", "delayed", "jobId", jobId, "delay", delayedTimestamp);
106
+ return 0
107
+ else
108
+ return -1
109
+ end
110
+ `;
111
+ exports.moveToDelayed = {
112
+ name: 'moveToDelayed',
113
+ content,
114
+ keys: 8,
115
+ };
116
+ //# sourceMappingURL=moveToDelayed-8.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"moveToDelayed-8.js","sourceRoot":"","sources":["../../../src/scripts/moveToDelayed-8.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CA0Gf,CAAC;AACW,QAAA,aAAa,GAAG;IAC3B,IAAI,EAAE,eAAe;IACrB,OAAO;IACP,IAAI,EAAE,CAAC;CACR,CAAC"}
@@ -134,41 +134,25 @@ end
134
134
  local function moveJobFromWaitToActive(keys, keyPrefix, jobId, processedOn, opts)
135
135
  -- Check if we need to perform rate limiting.
136
136
  local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
137
+ local expireTime
137
138
  if(maxJobs) then
138
139
  local rateLimiterKey = keys[6];
139
- local groupKey
140
- local groupKeyOpt = opts['limiter'] and opts['limiter']['groupKey'] or ""
141
- if groupKeyOpt ~= "" then
142
- groupKey = string.match(jobId, "[^:]+$")
143
- if groupKey ~= jobId then
144
- rateLimiterKey = rateLimiterKey .. ":" .. groupKey
145
- end
146
- end
147
- local jobCounter
148
- if groupKey ~= nil then
149
- if rateLimiterKey ~= keys[6] then
150
- jobCounter = tonumber(rcall("INCR", rateLimiterKey))
151
- end
152
- else
153
- jobCounter = tonumber(rcall("INCR", rateLimiterKey))
140
+ local jobCounter = tonumber(rcall("INCR", rateLimiterKey))
141
+ if jobCounter == 1 then
142
+ local limiterDuration = opts['limiter'] and opts['limiter']['duration']
143
+ rcall("PEXPIRE", rateLimiterKey, limiterDuration)
154
144
  end
155
- local limiterDuration = opts['limiter'] and opts['limiter']['duration']
156
- -- check if rate limit hit
157
- if jobCounter ~= nil and jobCounter > maxJobs then
158
- local exceedingJobs = jobCounter - maxJobs
159
- local expireTime = tonumber(rcall("PTTL", rateLimiterKey))
160
- local delay = expireTime + ((exceedingJobs - 1) * limiterDuration) / maxJobs;
161
- local timestamp = delay + tonumber(processedOn)
162
- -- put job into delayed queue
163
- rcall("ZADD", keys[7], timestamp * 0x1000 + bit.band(jobCounter, 0xfff), jobId);
164
- rcall("XADD", keys[4], "*", "event", "delayed", "jobId", jobId, "delay", timestamp);
165
- -- remove from active queue
145
+ -- check if we passed rate limit, we need to remove the job and return expireTime
146
+ if jobCounter > maxJobs then
147
+ expireTime = rcall("PTTL", rateLimiterKey)
148
+ -- remove from active queue and add back to the wait list
166
149
  rcall("LREM", keys[2], 1, jobId)
150
+ rcall("RPUSH", keys[1], jobId)
167
151
  -- Return when we can process more jobs
168
- return expireTime
152
+ return {0, 0, expireTime}
169
153
  else
170
- if jobCounter == 1 then
171
- rcall("PEXPIRE", rateLimiterKey, limiterDuration)
154
+ if jobCounter == maxJobs then
155
+ expireTime = rcall("PTTL", rateLimiterKey)
172
156
  end
173
157
  end
174
158
  end
@@ -182,7 +166,7 @@ local function moveJobFromWaitToActive(keys, keyPrefix, jobId, processedOn, opts
182
166
  rcall("XADD", keys[4], "*", "event", "active", "jobId", jobId, "prev", "waiting")
183
167
  rcall("HSET", jobKey, "processedOn", processedOn)
184
168
  rcall("HINCRBY", jobKey, "attemptsMade", 1)
185
- return {rcall("HGETALL", jobKey), jobId} -- get job data
169
+ return {rcall("HGETALL", jobKey), jobId, expireTime} -- get job data
186
170
  end
187
171
  --[[
188
172
  Function to recursively move from waitingChildren to failed.
@@ -266,11 +250,6 @@ local function promoteDelayedJobs(delayedKey, waitKey, priorityKey, pausedKey,
266
250
  rcall("HSET", prefix .. jobId, "delay", 0)
267
251
  end
268
252
  end
269
- local nextTimestamp = rcall("ZRANGE", delayedKey, 0, 0, "WITHSCORES")[2]
270
- if (nextTimestamp ~= nil) then
271
- nextTimestamp = nextTimestamp / 0x1000
272
- end
273
- return nextTimestamp
274
253
  end
275
254
  --[[
276
255
  Functions to remove jobs by max age.
@@ -421,6 +400,19 @@ local function updateParentDepsIfNeeded(parentKey, parentQueueKey, parentDepende
421
400
  rcall("XADD", parentQueueKey .. ":events", "*", "event", "waiting", "jobId", parentId, "prev", "waiting-children")
422
401
  end
423
402
  end
403
+ local function getRateLimitTTL(opts, limiterKey)
404
+ local maxJobs = tonumber(opts['limiter'] and opts['limiter']['max'])
405
+ if maxJobs then
406
+ local jobCounter = tonumber(rcall("GET", limiterKey))
407
+ if jobCounter ~= nil and jobCounter >= maxJobs then
408
+ local pttl = rcall("PTTL", KEYS[6])
409
+ if pttl > 0 then
410
+ return pttl
411
+ end
412
+ end
413
+ end
414
+ return 0
415
+ end
424
416
  local jobIdKey = KEYS[10]
425
417
  if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
426
418
  local opts = cmsgpack.unpack(ARGV[9])
@@ -507,6 +499,11 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
507
499
  if (ARGV[7] == "1") then
508
500
  -- Check if there are delayed jobs that can be promoted
509
501
  promoteDelayedJobs(KEYS[7], KEYS[1], KEYS[3], KEYS[8], KEYS[11], KEYS[4], ARGV[8], timestamp)
502
+ -- Check if we are rate limited first.
503
+ local pttl = getRateLimitTTL(opts, KEYS[6])
504
+ if pttl > 0 then
505
+ return { 0, 0, pttl }
506
+ end
510
507
  jobId = rcall("RPOPLPUSH", KEYS[1], KEYS[2])
511
508
  if jobId == "0" then
512
509
  rcall("LREM", KEYS[2], 1, 0)
@@ -518,7 +515,7 @@ if rcall("EXISTS", jobIdKey) == 1 then -- // Make sure job exists
518
515
  if (nextTimestamp ~= nil) then
519
516
  -- The result is guaranteed to be positive, since the
520
517
  -- ZRANGEBYSCORE command would have return a job otherwise.
521
- return nextTimestamp - timestamp
518
+ return {0, 0, 0, nextTimestamp - timestamp}
522
519
  end
523
520
  end
524
521
  local waitLen = rcall("LLEN", KEYS[1])
@@ -1 +1 @@
1
- {"version":3,"file":"moveToFinished-12.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-12.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAmhBf,CAAC;AACW,QAAA,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}
1
+ {"version":3,"file":"moveToFinished-12.js","sourceRoot":"","sources":["../../../src/scripts/moveToFinished-12.ts"],"names":[],"mappings":";;;AAAA,MAAM,OAAO,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAghBf,CAAC;AACW,QAAA,cAAc,GAAG;IAC5B,IAAI,EAAE,gBAAgB;IACtB,OAAO;IACP,IAAI,EAAE,EAAE;CACT,CAAC"}