sidekiq-throttled 0.11.0 → 0.15.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +51 -0
- data/.rubocop/layout.yml +24 -0
- data/.rubocop/lint.yml +41 -0
- data/.rubocop/metrics.yml +4 -0
- data/.rubocop/performance.yml +25 -0
- data/.rubocop/rspec.yml +3 -0
- data/.rubocop/style.yml +84 -0
- data/.rubocop.yml +13 -68
- data/.rubocop_todo.yml +10 -35
- data/.travis.yml +2 -1
- data/Appraisals +16 -0
- data/CHANGES.md +53 -0
- data/Gemfile +6 -5
- data/LICENSE.md +2 -1
- data/README.md +49 -18
- data/gemfiles/sidekiq_5.0.gemfile +6 -5
- data/gemfiles/sidekiq_5.1.gemfile +6 -5
- data/gemfiles/sidekiq_5.2.gemfile +6 -5
- data/gemfiles/sidekiq_6.0.gemfile +31 -0
- data/gemfiles/sidekiq_6.1.gemfile +31 -0
- data/gemfiles/sidekiq_6.2.gemfile +31 -0
- data/gemfiles/sidekiq_6.3.gemfile +31 -0
- data/lib/sidekiq/throttled/communicator/callbacks.rb +8 -9
- data/lib/sidekiq/throttled/communicator/listener.rb +3 -3
- data/lib/sidekiq/throttled/expirable_list.rb +1 -1
- data/lib/sidekiq/throttled/fetch.rb +34 -20
- data/lib/sidekiq/throttled/queue_name.rb +1 -1
- data/lib/sidekiq/throttled/registry.rb +2 -5
- data/lib/sidekiq/throttled/strategy/concurrency.rb +5 -5
- data/lib/sidekiq/throttled/strategy/threshold.rb +6 -5
- data/lib/sidekiq/throttled/strategy.rb +15 -20
- data/lib/sidekiq/throttled/strategy_collection.rb +69 -0
- data/lib/sidekiq/throttled/utils.rb +1 -1
- data/lib/sidekiq/throttled/version.rb +1 -1
- data/lib/sidekiq/throttled/web/stats.rb +2 -6
- data/lib/sidekiq/throttled/web/summary_fix.rb +3 -2
- data/lib/sidekiq/throttled/web/throttled.html.erb +6 -2
- data/lib/sidekiq/throttled/web.rb +1 -1
- data/lib/sidekiq/throttled/worker.rb +2 -2
- data/lib/sidekiq/throttled.rb +14 -2
- data/sidekiq-throttled.gemspec +3 -1
- metadata +21 -9
data/README.md
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
# Sidekiq::Throttled
|
2
2
|
|
3
|
-
[](http://inch-ci.org/github/sensortower/sidekiq-throttled)
|
3
|
+
[](http://rubygems.org/gems/sidekiq-throttled)
|
4
|
+
[](https://github.com/sensortower/sidekiq-throttled/actions?query=workflow%3ACI+branch%3Amaster)
|
5
|
+
[](https://codeclimate.com/github/sensortower/sidekiq-throttled)
|
6
|
+
[](https://coveralls.io/github/sensortower/sidekiq-throttled?branch=master)
|
7
|
+
[](http://inch-ci.org/github/sensortower/sidekiq-throttled)
|
8
|
+
[](http://www.rubydoc.info/gems/sidekiq-throttled)
|
8
9
|
|
9
10
|
Concurrency and threshold throttling for [Sidekiq][sidekiq].
|
10
11
|
|
11
|
-
|
12
12
|
## Installation
|
13
13
|
|
14
14
|
Add this line to your application's Gemfile:
|
@@ -36,6 +36,9 @@ require "sidekiq/throttled"
|
|
36
36
|
Sidekiq::Throttled.setup!
|
37
37
|
```
|
38
38
|
|
39
|
+
Load order can be an issue if you are using other Sidekiq plugins and/or middleware.
|
40
|
+
To prevent any problems, add the `.setup!` call to the bottom of your init file.
|
41
|
+
|
39
42
|
Once you've done that you can include `Sidekiq::Throttled::Worker` to your
|
40
43
|
job classes and configure throttling:
|
41
44
|
|
@@ -46,12 +49,12 @@ class MyWorker
|
|
46
49
|
|
47
50
|
sidekiq_options :queue => :my_queue
|
48
51
|
|
49
|
-
sidekiq_throttle(
|
52
|
+
sidekiq_throttle(
|
50
53
|
# Allow maximum 10 concurrent jobs of this class at a time.
|
51
54
|
:concurrency => { :limit => 10 },
|
52
55
|
# Allow maximum 1K jobs being processed within one hour window.
|
53
56
|
:threshold => { :limit => 1_000, :period => 1.hour }
|
54
|
-
|
57
|
+
)
|
55
58
|
|
56
59
|
def perform
|
57
60
|
# ...
|
@@ -75,11 +78,11 @@ class MyWorker
|
|
75
78
|
|
76
79
|
sidekiq_options :queue => :my_queue
|
77
80
|
|
78
|
-
sidekiq_throttle(
|
81
|
+
sidekiq_throttle(
|
79
82
|
:concurrency => { :limit => 10 },
|
80
83
|
:threshold => { :limit => 100, :period => 1.hour }
|
81
84
|
:observer => MY_OBSERVER
|
82
|
-
|
85
|
+
)
|
83
86
|
|
84
87
|
def perform(*args)
|
85
88
|
# ...
|
@@ -88,7 +91,7 @@ end
|
|
88
91
|
```
|
89
92
|
|
90
93
|
Observer will receive `strategy, *args` arguments, where `strategy` is a Symbol
|
91
|
-
`:concurrency` or `:threshold`, and `*args` are the
|
94
|
+
`:concurrency` or `:threshold`, and `*args` are the arguments that were passed
|
92
95
|
to the job.
|
93
96
|
|
94
97
|
|
@@ -103,10 +106,10 @@ class MyWorker
|
|
103
106
|
|
104
107
|
sidekiq_options :queue => :my_queue
|
105
108
|
|
106
|
-
sidekiq_throttle(
|
109
|
+
sidekiq_throttle(
|
107
110
|
# Allow maximum 10 concurrent jobs per user at a time.
|
108
111
|
:concurrency => { :limit => 10, :key_suffix => -> (user_id) { user_id } }
|
109
|
-
|
112
|
+
)
|
110
113
|
|
111
114
|
def perform(user_id)
|
112
115
|
# ...
|
@@ -125,7 +128,7 @@ class MyWorker
|
|
125
128
|
|
126
129
|
sidekiq_options :queue => :my_queue
|
127
130
|
|
128
|
-
sidekiq_throttle(
|
131
|
+
sidekiq_throttle(
|
129
132
|
# Allow maximum 1000 concurrent jobs of this class at a time for VIPs and 10 for all other users.
|
130
133
|
:concurrency => {
|
131
134
|
:limit => ->(user_id) { User.vip?(user_id) ? 1_000 : 10 },
|
@@ -136,7 +139,7 @@ class MyWorker
|
|
136
139
|
:limit => ->(user_id) { User.vip?(user_id) ? 1_000 : 10 },
|
137
140
|
:period => ->(user_id) { User.vip?(user_id) ? 1.hour : 1.day },
|
138
141
|
:key_suffix => ->(user_id) { User.vip?(user_id) ? "vip" : "std" }
|
139
|
-
|
142
|
+
)
|
140
143
|
|
141
144
|
def perform(user_id)
|
142
145
|
# ...
|
@@ -144,6 +147,30 @@ class MyWorker
|
|
144
147
|
end
|
145
148
|
```
|
146
149
|
|
150
|
+
You also can use several different keys to throttle one worker.
|
151
|
+
|
152
|
+
``` ruby
|
153
|
+
class MyWorker
|
154
|
+
include Sidekiq::Worker
|
155
|
+
include Sidekiq::Throttled::Worker
|
156
|
+
|
157
|
+
sidekiq_options :queue => :my_queue
|
158
|
+
|
159
|
+
sidekiq_throttle(
|
160
|
+
# Allow maximum 10 concurrent jobs per project at a time and maximum 2 jobs per user
|
161
|
+
:concurrency => [
|
162
|
+
{ :limit => 10, :key_suffix => -> (project_id, user_id) { project_id } },
|
163
|
+
{ :limit => 2, :key_suffix => -> (project_id, user_id) { user_id } }
|
164
|
+
]
|
165
|
+
# For :threshold it works the same
|
166
|
+
)
|
167
|
+
|
168
|
+
def perform(project_id, user_id)
|
169
|
+
# ...
|
170
|
+
end
|
171
|
+
end
|
172
|
+
```
|
173
|
+
|
147
174
|
**NB** Don't forget to specify `:key_suffix` and make it return different values
|
148
175
|
if you are using dynamic limit/period options. Otherwise you risk getting into
|
149
176
|
some trouble.
|
@@ -206,9 +233,9 @@ end
|
|
206
233
|
This library aims to support and is [tested against][travis] the following Ruby
|
207
234
|
versions:
|
208
235
|
|
209
|
-
* Ruby 2.4.x
|
210
|
-
* Ruby 2.5.x
|
211
236
|
* Ruby 2.6.x
|
237
|
+
* Ruby 2.7.x
|
238
|
+
* Ruby 3.0.x
|
212
239
|
|
213
240
|
If something doesn't work on one of these versions, it's a bug.
|
214
241
|
|
@@ -231,6 +258,10 @@ This library aims to support work with following [Sidekiq][sidekiq] versions:
|
|
231
258
|
* Sidekiq 5.0.x
|
232
259
|
* Sidekiq 5.1.x
|
233
260
|
* Sidekiq 5.2.x
|
261
|
+
* Sidekiq 6.0.x
|
262
|
+
* Sidekiq 6.1.x
|
263
|
+
* Sidekiq 6.2.x
|
264
|
+
* Sidekiq 6.3.x
|
234
265
|
|
235
266
|
|
236
267
|
## Contributing
|
@@ -258,7 +289,7 @@ Don't forget to run `appraisal update` after any changes to `Gemfile`.
|
|
258
289
|
|
259
290
|
## Copyright
|
260
291
|
|
261
|
-
Copyright (c)
|
292
|
+
Copyright (c) 2020-2021 Alexey Zapparov, SensorTower Inc.
|
262
293
|
See LICENSE.md for further details.
|
263
294
|
|
264
295
|
|
@@ -5,8 +5,9 @@ source "https://rubygems.org"
|
|
5
5
|
gem "appraisal"
|
6
6
|
gem "rake"
|
7
7
|
gem "rspec"
|
8
|
-
gem "rubocop", "~> 0.
|
9
|
-
gem "rubocop-
|
8
|
+
gem "rubocop", "~> 0.90.0", require: false
|
9
|
+
gem "rubocop-performance", "~> 1.8.0", require: false
|
10
|
+
gem "rubocop-rspec", "~> 1.43.2", require: false
|
10
11
|
gem "sidekiq", "~> 5.0.0"
|
11
12
|
|
12
13
|
group :development do
|
@@ -17,13 +18,13 @@ group :development do
|
|
17
18
|
end
|
18
19
|
|
19
20
|
group :test do
|
21
|
+
gem "apparition"
|
20
22
|
gem "capybara"
|
21
23
|
gem "coveralls", require: false
|
22
|
-
gem "poltergeist"
|
23
24
|
gem "puma"
|
24
25
|
gem "rack-test"
|
25
|
-
gem "simplecov"
|
26
|
-
gem "sinatra"
|
26
|
+
gem "simplecov"
|
27
|
+
gem "sinatra"
|
27
28
|
gem "timecop"
|
28
29
|
end
|
29
30
|
|
@@ -5,8 +5,9 @@ source "https://rubygems.org"
|
|
5
5
|
gem "appraisal"
|
6
6
|
gem "rake"
|
7
7
|
gem "rspec"
|
8
|
-
gem "rubocop", "~> 0.
|
9
|
-
gem "rubocop-
|
8
|
+
gem "rubocop", "~> 0.90.0", require: false
|
9
|
+
gem "rubocop-performance", "~> 1.8.0", require: false
|
10
|
+
gem "rubocop-rspec", "~> 1.43.2", require: false
|
10
11
|
gem "sidekiq", "~> 5.1.0"
|
11
12
|
|
12
13
|
group :development do
|
@@ -17,13 +18,13 @@ group :development do
|
|
17
18
|
end
|
18
19
|
|
19
20
|
group :test do
|
21
|
+
gem "apparition"
|
20
22
|
gem "capybara"
|
21
23
|
gem "coveralls", require: false
|
22
|
-
gem "poltergeist"
|
23
24
|
gem "puma"
|
24
25
|
gem "rack-test"
|
25
|
-
gem "simplecov"
|
26
|
-
gem "sinatra"
|
26
|
+
gem "simplecov"
|
27
|
+
gem "sinatra"
|
27
28
|
gem "timecop"
|
28
29
|
end
|
29
30
|
|
@@ -5,8 +5,9 @@ source "https://rubygems.org"
|
|
5
5
|
gem "appraisal"
|
6
6
|
gem "rake"
|
7
7
|
gem "rspec"
|
8
|
-
gem "rubocop", "~> 0.
|
9
|
-
gem "rubocop-
|
8
|
+
gem "rubocop", "~> 0.90.0", require: false
|
9
|
+
gem "rubocop-performance", "~> 1.8.0", require: false
|
10
|
+
gem "rubocop-rspec", "~> 1.43.2", require: false
|
10
11
|
gem "sidekiq", "~> 5.2.0"
|
11
12
|
|
12
13
|
group :development do
|
@@ -17,13 +18,13 @@ group :development do
|
|
17
18
|
end
|
18
19
|
|
19
20
|
group :test do
|
21
|
+
gem "apparition"
|
20
22
|
gem "capybara"
|
21
23
|
gem "coveralls", require: false
|
22
|
-
gem "poltergeist"
|
23
24
|
gem "puma"
|
24
25
|
gem "rack-test"
|
25
|
-
gem "simplecov"
|
26
|
-
gem "sinatra"
|
26
|
+
gem "simplecov"
|
27
|
+
gem "sinatra"
|
27
28
|
gem "timecop"
|
28
29
|
end
|
29
30
|
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# This file was generated by Appraisal
|
2
|
+
|
3
|
+
source "https://rubygems.org"
|
4
|
+
|
5
|
+
gem "appraisal"
|
6
|
+
gem "rake"
|
7
|
+
gem "rspec"
|
8
|
+
gem "rubocop", "~> 0.90.0", require: false
|
9
|
+
gem "rubocop-performance", "~> 1.8.0", require: false
|
10
|
+
gem "rubocop-rspec", "~> 1.43.2", require: false
|
11
|
+
gem "sidekiq", "~> 6.0.0"
|
12
|
+
|
13
|
+
group :development do
|
14
|
+
gem "byebug"
|
15
|
+
gem "guard", require: false
|
16
|
+
gem "guard-rspec", require: false
|
17
|
+
gem "guard-rubocop", require: false
|
18
|
+
end
|
19
|
+
|
20
|
+
group :test do
|
21
|
+
gem "apparition"
|
22
|
+
gem "capybara"
|
23
|
+
gem "coveralls", require: false
|
24
|
+
gem "puma"
|
25
|
+
gem "rack-test"
|
26
|
+
gem "simplecov"
|
27
|
+
gem "sinatra"
|
28
|
+
gem "timecop"
|
29
|
+
end
|
30
|
+
|
31
|
+
gemspec path: "../"
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# This file was generated by Appraisal
|
2
|
+
|
3
|
+
source "https://rubygems.org"
|
4
|
+
|
5
|
+
gem "appraisal"
|
6
|
+
gem "rake"
|
7
|
+
gem "rspec"
|
8
|
+
gem "rubocop", "~> 0.90.0", require: false
|
9
|
+
gem "rubocop-performance", "~> 1.8.0", require: false
|
10
|
+
gem "rubocop-rspec", "~> 1.43.2", require: false
|
11
|
+
gem "sidekiq", "~> 6.1.0"
|
12
|
+
|
13
|
+
group :development do
|
14
|
+
gem "byebug"
|
15
|
+
gem "guard", require: false
|
16
|
+
gem "guard-rspec", require: false
|
17
|
+
gem "guard-rubocop", require: false
|
18
|
+
end
|
19
|
+
|
20
|
+
group :test do
|
21
|
+
gem "apparition"
|
22
|
+
gem "capybara"
|
23
|
+
gem "coveralls", require: false
|
24
|
+
gem "puma"
|
25
|
+
gem "rack-test"
|
26
|
+
gem "simplecov"
|
27
|
+
gem "sinatra"
|
28
|
+
gem "timecop"
|
29
|
+
end
|
30
|
+
|
31
|
+
gemspec path: "../"
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# This file was generated by Appraisal
|
2
|
+
|
3
|
+
source "https://rubygems.org"
|
4
|
+
|
5
|
+
gem "appraisal"
|
6
|
+
gem "rake"
|
7
|
+
gem "rspec"
|
8
|
+
gem "rubocop", "~> 0.90.0", require: false
|
9
|
+
gem "rubocop-performance", "~> 1.8.0", require: false
|
10
|
+
gem "rubocop-rspec", "~> 1.43.2", require: false
|
11
|
+
gem "sidekiq", "~> 6.2.0"
|
12
|
+
|
13
|
+
group :development do
|
14
|
+
gem "byebug"
|
15
|
+
gem "guard", require: false
|
16
|
+
gem "guard-rspec", require: false
|
17
|
+
gem "guard-rubocop", require: false
|
18
|
+
end
|
19
|
+
|
20
|
+
group :test do
|
21
|
+
gem "apparition"
|
22
|
+
gem "capybara"
|
23
|
+
gem "coveralls", require: false
|
24
|
+
gem "puma"
|
25
|
+
gem "rack-test"
|
26
|
+
gem "simplecov"
|
27
|
+
gem "sinatra"
|
28
|
+
gem "timecop"
|
29
|
+
end
|
30
|
+
|
31
|
+
gemspec path: "../"
|
@@ -0,0 +1,31 @@
|
|
1
|
+
# This file was generated by Appraisal
|
2
|
+
|
3
|
+
source "https://rubygems.org"
|
4
|
+
|
5
|
+
gem "appraisal"
|
6
|
+
gem "rake"
|
7
|
+
gem "rspec"
|
8
|
+
gem "rubocop", "~> 0.90.0", require: false
|
9
|
+
gem "rubocop-performance", "~> 1.8.0", require: false
|
10
|
+
gem "rubocop-rspec", "~> 1.43.2", require: false
|
11
|
+
gem "sidekiq", "~> 6.3.0"
|
12
|
+
|
13
|
+
group :development do
|
14
|
+
gem "byebug"
|
15
|
+
gem "guard", require: false
|
16
|
+
gem "guard-rspec", require: false
|
17
|
+
gem "guard-rubocop", require: false
|
18
|
+
end
|
19
|
+
|
20
|
+
group :test do
|
21
|
+
gem "apparition"
|
22
|
+
gem "capybara"
|
23
|
+
gem "coveralls", require: false
|
24
|
+
gem "puma"
|
25
|
+
gem "rack-test"
|
26
|
+
gem "simplecov"
|
27
|
+
gem "sinatra"
|
28
|
+
gem "timecop"
|
29
|
+
end
|
30
|
+
|
31
|
+
gemspec path: "../"
|
@@ -43,6 +43,7 @@ module Sidekiq
|
|
43
43
|
# @return [self]
|
44
44
|
def on(event, &handler)
|
45
45
|
raise ArgumentError, "No block given" unless handler
|
46
|
+
|
46
47
|
@mutex.synchronize { @handlers[event.to_s] << handler }
|
47
48
|
self
|
48
49
|
end
|
@@ -54,17 +55,15 @@ module Sidekiq
|
|
54
55
|
# @return [void]
|
55
56
|
def run(event, payload = nil)
|
56
57
|
@mutex.synchronize do
|
57
|
-
Fiber.new do
|
58
|
+
fiber = Fiber.new do
|
58
59
|
@handlers[event.to_s].each do |callback|
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
handle_exception(e, {
|
63
|
-
:context => "sidekiq:throttled"
|
64
|
-
})
|
65
|
-
end
|
60
|
+
callback.call(payload)
|
61
|
+
rescue => e
|
62
|
+
handle_exception(e, :context => "sidekiq:throttled")
|
66
63
|
end
|
67
|
-
end
|
64
|
+
end
|
65
|
+
|
66
|
+
fiber.resume
|
68
67
|
end
|
69
68
|
end
|
70
69
|
end
|
@@ -62,7 +62,7 @@ module Sidekiq
|
|
62
62
|
# - `Exception` is recorded to the log and re-raised.
|
63
63
|
#
|
64
64
|
# @return [void]
|
65
|
-
def listen
|
65
|
+
def listen # rubocop:disable Metrics/MethodLength
|
66
66
|
subscribe
|
67
67
|
rescue Sidekiq::Shutdown
|
68
68
|
@terminated = true
|
@@ -88,7 +88,7 @@ module Sidekiq
|
|
88
88
|
# @see http://redis.io/commands/subscribe
|
89
89
|
# @see Callbacks#run
|
90
90
|
# @return [void]
|
91
|
-
def subscribe
|
91
|
+
def subscribe # rubocop:disable Metrics/MethodLength
|
92
92
|
Sidekiq.redis do |conn|
|
93
93
|
conn.subscribe @channel do |on|
|
94
94
|
on.subscribe do
|
@@ -97,7 +97,7 @@ module Sidekiq
|
|
97
97
|
end
|
98
98
|
|
99
99
|
on.message do |_channel, data|
|
100
|
-
message, payload = Marshal.load(data)
|
100
|
+
message, payload = Marshal.load(data) # rubocop:disable Security/MarshalLoad:
|
101
101
|
@callbacks.run("message:#{message}", payload)
|
102
102
|
end
|
103
103
|
end
|
@@ -11,7 +11,7 @@ module Sidekiq
|
|
11
11
|
#
|
12
12
|
# ## Implementation
|
13
13
|
#
|
14
|
-
# Internally list holds an array of arrays. Thus
|
14
|
+
# Internally list holds an array of arrays. Thus each element is a tuple of
|
15
15
|
# monotonic timestamp (when element was added) and element itself:
|
16
16
|
#
|
17
17
|
# [
|
@@ -12,16 +12,47 @@ module Sidekiq
|
|
12
12
|
#
|
13
13
|
# @private
|
14
14
|
class Fetch
|
15
|
+
module BulkRequeue
|
16
|
+
# Requeues all given units as a single operation.
|
17
|
+
#
|
18
|
+
# @see http://www.rubydoc.info/github/redis/redis-rb/master/Redis#pipelined-instance_method
|
19
|
+
# @param [Array<Fetch::UnitOfWork>] units
|
20
|
+
# @return [void]
|
21
|
+
def bulk_requeue(units, _options)
|
22
|
+
return if units.empty?
|
23
|
+
|
24
|
+
Sidekiq.logger.debug { "Re-queueing terminated jobs" }
|
25
|
+
Sidekiq.redis { |conn| conn.pipelined { units.each(&:requeue) } }
|
26
|
+
Sidekiq.logger.info("Pushed #{units.size} jobs back to Redis")
|
27
|
+
rescue => e
|
28
|
+
Sidekiq.logger.warn("Failed to requeue #{units.size} jobs: #{e}")
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
# https://github.com/mperham/sidekiq/commit/fce05c9d4b4c0411c982078a4cf3a63f20f739bc
|
33
|
+
if Gem::Version.new(Sidekiq::VERSION) < Gem::Version.new("6.1.0")
|
34
|
+
extend BulkRequeue
|
35
|
+
else
|
36
|
+
include BulkRequeue
|
37
|
+
end
|
15
38
|
# Timeout to sleep between fetch retries in case of no job received,
|
16
39
|
# as well as timeout to wait for redis to give us something to work.
|
17
40
|
TIMEOUT = 2
|
18
41
|
|
19
42
|
# Initializes fetcher instance.
|
43
|
+
# @param options [Hash]
|
44
|
+
# @option options [Integer] :throttled_queue_cooldown (TIMEOUT)
|
45
|
+
# Min delay in seconds before queue will be polled again after
|
46
|
+
# throttled job.
|
47
|
+
# @option options [Boolean] :strict (false)
|
48
|
+
# @option options [Array<#to_s>] :queue
|
20
49
|
def initialize(options)
|
21
|
-
@paused = ExpirableList.new(TIMEOUT)
|
50
|
+
@paused = ExpirableList.new(options.fetch(:throttled_queue_cooldown, TIMEOUT))
|
22
51
|
|
23
|
-
@strict = options
|
24
|
-
@queues = options
|
52
|
+
@strict = options.fetch(:strict, false)
|
53
|
+
@queues = options.fetch(:queues).map { |q| QueueName.expand q }
|
54
|
+
|
55
|
+
raise ArgumentError, "empty :queues" if @queues.empty?
|
25
56
|
|
26
57
|
@queues.uniq! if @strict
|
27
58
|
end
|
@@ -42,23 +73,6 @@ module Sidekiq
|
|
42
73
|
nil
|
43
74
|
end
|
44
75
|
|
45
|
-
class << self
|
46
|
-
# Requeues all given units as a single operation.
|
47
|
-
#
|
48
|
-
# @see http://www.rubydoc.info/github/redis/redis-rb/master/Redis#pipelined-instance_method
|
49
|
-
# @param [Array<Fetch::UnitOfWork>] units
|
50
|
-
# @return [void]
|
51
|
-
def bulk_requeue(units, _options)
|
52
|
-
return if units.empty?
|
53
|
-
|
54
|
-
Sidekiq.logger.debug { "Re-queueing terminated jobs" }
|
55
|
-
Sidekiq.redis { |conn| conn.pipelined { units.each(&:requeue) } }
|
56
|
-
Sidekiq.logger.info("Pushed #{units.size} jobs back to Redis")
|
57
|
-
rescue => e
|
58
|
-
Sidekiq.logger.warn("Failed to requeue #{units.size} jobs: #{e}")
|
59
|
-
end
|
60
|
-
end
|
61
|
-
|
62
76
|
private
|
63
77
|
|
64
78
|
# Tries to pop pair of `queue` and job `message` out of sidekiq queues.
|
@@ -18,20 +18,16 @@ module Sidekiq
|
|
18
18
|
|
19
19
|
# Adds strategy to the registry.
|
20
20
|
#
|
21
|
-
# @note prints a warning to STDERR upon duplicate strategy name
|
22
21
|
# @param (see Strategy#initialize)
|
23
22
|
# @return [Strategy]
|
24
23
|
def add(name, **kwargs)
|
25
24
|
name = name.to_s
|
26
25
|
|
27
|
-
warn "Duplicate strategy name: #{name}" if @strategies[name]
|
28
|
-
|
29
26
|
@strategies[name] = Strategy.new(name, **kwargs)
|
30
27
|
end
|
31
28
|
|
32
29
|
# Adds alias for existing strategy.
|
33
30
|
#
|
34
|
-
# @note prints a warning to STDERR upon duplicate strategy name
|
35
31
|
# @param (#to_s) new_name
|
36
32
|
# @param (#to_s) old_name
|
37
33
|
# @raise [RuntimeError] if no strategy found with `old_name`
|
@@ -40,7 +36,6 @@ module Sidekiq
|
|
40
36
|
new_name = new_name.to_s
|
41
37
|
old_name = old_name.to_s
|
42
38
|
|
43
|
-
warn "Duplicate strategy name: #{new_name}" if @strategies[new_name]
|
44
39
|
raise "Strategy not found: #{old_name}" unless @strategies[old_name]
|
45
40
|
|
46
41
|
@aliases[new_name] = @strategies[old_name]
|
@@ -74,6 +69,7 @@ module Sidekiq
|
|
74
69
|
# @return [Registry]
|
75
70
|
def each
|
76
71
|
return to_enum(__method__) unless block_given?
|
72
|
+
|
77
73
|
@strategies.each { |*args| yield(*args) }
|
78
74
|
self
|
79
75
|
end
|
@@ -88,6 +84,7 @@ module Sidekiq
|
|
88
84
|
# @return [Registry]
|
89
85
|
def each_with_static_keys
|
90
86
|
return to_enum(__method__) unless block_given?
|
87
|
+
|
91
88
|
@strategies.each do |name, strategy|
|
92
89
|
yield(name, strategy) unless strategy.dynamic?
|
93
90
|
end
|
@@ -46,12 +46,12 @@ module Sidekiq
|
|
46
46
|
return false unless job_limit
|
47
47
|
return true if job_limit <= 0
|
48
48
|
|
49
|
-
|
50
|
-
|
51
|
-
:argv => [jid.to_s, job_limit, @ttl, Time.now.to_f]
|
52
|
-
}
|
49
|
+
keys = [key(job_args)]
|
50
|
+
argv = [jid.to_s, job_limit, @ttl, Time.now.to_f]
|
53
51
|
|
54
|
-
Sidekiq.redis
|
52
|
+
Sidekiq.redis do |redis|
|
53
|
+
1 == SCRIPT.eval(redis, :keys => keys, :argv => argv)
|
54
|
+
end
|
55
55
|
end
|
56
56
|
|
57
57
|
# @return [Integer] Current count of jobs
|
@@ -48,6 +48,7 @@ module Sidekiq
|
|
48
48
|
# @return [Float] Period in seconds
|
49
49
|
def period(job_args = nil)
|
50
50
|
return @period.to_f unless @period.respond_to? :call
|
51
|
+
|
51
52
|
@period.call(*job_args).to_f
|
52
53
|
end
|
53
54
|
|
@@ -62,12 +63,12 @@ module Sidekiq
|
|
62
63
|
return false unless job_limit
|
63
64
|
return true if job_limit <= 0
|
64
65
|
|
65
|
-
|
66
|
-
|
67
|
-
:argv => [job_limit, period(job_args), Time.now.to_f]
|
68
|
-
}
|
66
|
+
keys = [key(job_args)]
|
67
|
+
argv = [job_limit, period(job_args), Time.now.to_f]
|
69
68
|
|
70
|
-
Sidekiq.redis
|
69
|
+
Sidekiq.redis do |redis|
|
70
|
+
1 == SCRIPT.eval(redis, :keys => keys, :argv => argv)
|
71
|
+
end
|
71
72
|
end
|
72
73
|
|
73
74
|
# @return [Integer] Current count of jobs
|