sidekiq-throttled 0.16.2 → 1.4.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.adoc +332 -0
- data/lib/sidekiq/throttled/config.rb +44 -0
- data/lib/sidekiq/throttled/cooldown.rb +55 -0
- data/lib/sidekiq/throttled/expirable_set.rb +70 -0
- data/lib/sidekiq/throttled/job.rb +4 -4
- data/lib/sidekiq/throttled/message.rb +32 -0
- data/lib/sidekiq/throttled/middlewares/server.rb +28 -0
- data/lib/sidekiq/throttled/patches/basic_fetch.rb +45 -0
- data/lib/sidekiq/throttled/patches/super_fetch.rb +52 -0
- data/lib/sidekiq/throttled/patches/throttled_retriever.rb +26 -0
- data/lib/sidekiq/throttled/registry.rb +4 -7
- data/lib/sidekiq/throttled/strategy/base.rb +1 -1
- data/lib/sidekiq/throttled/strategy/concurrency.rb +4 -6
- data/lib/sidekiq/throttled/strategy/threshold.rb +4 -6
- data/lib/sidekiq/throttled/strategy.rb +10 -10
- data/lib/sidekiq/throttled/strategy_collection.rb +8 -9
- data/lib/sidekiq/throttled/version.rb +1 -1
- data/lib/sidekiq/throttled/web.rb +2 -45
- data/lib/sidekiq/throttled/worker.rb +1 -1
- data/lib/sidekiq/throttled.rb +46 -67
- metadata +27 -73
- data/.coveralls.yml +0 -1
- data/.github/dependabot.yml +0 -12
- data/.github/workflows/ci.yml +0 -52
- data/.gitignore +0 -12
- data/.rspec +0 -5
- data/.rubocop.yml +0 -20
- data/.rubocop_todo.yml +0 -68
- data/.travis.yml +0 -39
- data/.yardopts +0 -1
- data/Appraisals +0 -25
- data/CHANGES.md +0 -300
- data/Gemfile +0 -34
- data/Guardfile +0 -25
- data/README.md +0 -301
- data/Rakefile +0 -27
- data/gemfiles/sidekiq_6.0.gemfile +0 -33
- data/gemfiles/sidekiq_6.1.gemfile +0 -33
- data/gemfiles/sidekiq_6.2.gemfile +0 -33
- data/gemfiles/sidekiq_6.3.gemfile +0 -33
- data/gemfiles/sidekiq_6.4.gemfile +0 -33
- data/gemfiles/sidekiq_6.5.gemfile +0 -33
- data/lib/sidekiq/throttled/communicator/callbacks.rb +0 -72
- data/lib/sidekiq/throttled/communicator/exception_handler.rb +0 -25
- data/lib/sidekiq/throttled/communicator/listener.rb +0 -109
- data/lib/sidekiq/throttled/communicator.rb +0 -116
- data/lib/sidekiq/throttled/configuration.rb +0 -50
- data/lib/sidekiq/throttled/expirable_list.rb +0 -70
- data/lib/sidekiq/throttled/fetch/unit_of_work.rb +0 -83
- data/lib/sidekiq/throttled/fetch.rb +0 -107
- data/lib/sidekiq/throttled/middleware.rb +0 -22
- data/lib/sidekiq/throttled/patches/queue.rb +0 -18
- data/lib/sidekiq/throttled/queue_name.rb +0 -46
- data/lib/sidekiq/throttled/queues_pauser.rb +0 -152
- data/lib/sidekiq/throttled/testing.rb +0 -12
- data/lib/sidekiq/throttled/utils.rb +0 -19
- data/lib/sidekiq/throttled/web/queues.html.erb +0 -49
- data/lib/sidekiq/throttled/web/summary_fix.js +0 -10
- data/lib/sidekiq/throttled/web/summary_fix.rb +0 -35
- data/rubocop/layout.yml +0 -24
- data/rubocop/lint.yml +0 -41
- data/rubocop/metrics.yml +0 -4
- data/rubocop/performance.yml +0 -25
- data/rubocop/rspec.yml +0 -3
- data/rubocop/style.yml +0 -84
- data/sidekiq-throttled.gemspec +0 -36
- /data/{LICENSE.md → LICENSE.txt} +0 -0
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 7746a966e9feb8f571468c3f9f790b618faacdb6a6b405183acce7afc9210fb0
|
4
|
+
data.tar.gz: 7b4a320e9ed8bab632e7834f67015b6b377071c77ac0838cf2a2db662c34ec81
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ed36a36f934b607f08c132db2296a02741d4fad2a4436f4c475bc66f6a0cda49ece9e95782092603e4a3146de4c9b4dc7eec95223d4ba15a3c92715d861a2c91
|
7
|
+
data.tar.gz: c99428f6499ddcc70d89c9b73c8dcbcae275a6a81533645dcc7d8d0d0a7f04492b876afb4d651a934e5df2ad82996925b1f551ff16a9309beba99a146c0726f9
|
data/README.adoc
ADDED
@@ -0,0 +1,332 @@
|
|
1
|
+
= Sidekiq::Throttled
|
2
|
+
:ci-link: https://github.com/ixti/sidekiq-throttled/actions/workflows/ci.yml
|
3
|
+
:ci-badge: https://img.shields.io/github/actions/workflow/status/ixti/sidekiq-throttled/ci.yml?branch=main&style=for-the-badge
|
4
|
+
:gem-link: http://rubygems.org/gems/sidekiq-throttled
|
5
|
+
:gem-badge: https://img.shields.io/gem/v/sidekiq-throttled?style=for-the-badge
|
6
|
+
:doc-link: http://www.rubydoc.info/gems/sidekiq-throttled
|
7
|
+
:doc-badge: https://img.shields.io/badge/Documentation-API-blue?style=for-the-badge
|
8
|
+
|
9
|
+
****
|
10
|
+
{ci-link}[image:{ci-badge}[CI Status]]
|
11
|
+
{gem-link}[image:{gem-badge}[Latest Version]]
|
12
|
+
{doc-link}[image:{doc-badge}[API Documentation]]
|
13
|
+
****
|
14
|
+
|
15
|
+
Concurrency and threshold throttling for https://github.com/sidekiq/sidekiq[Sidekiq].
|
16
|
+
|
17
|
+
== Installation
|
18
|
+
|
19
|
+
Add this line to your application's Gemfile:
|
20
|
+
|
21
|
+
[source,ruby]
|
22
|
+
----
|
23
|
+
gem "sidekiq-throttled"
|
24
|
+
----
|
25
|
+
|
26
|
+
And then execute:
|
27
|
+
|
28
|
+
$ bundle
|
29
|
+
|
30
|
+
Or install it yourself as:
|
31
|
+
|
32
|
+
$ gem install sidekiq-throttled
|
33
|
+
|
34
|
+
|
35
|
+
== Usage
|
36
|
+
|
37
|
+
Add somewhere in your app's bootstrap (e.g. `config/initializers/sidekiq.rb` if
|
38
|
+
you are using Rails):
|
39
|
+
|
40
|
+
[source,ruby]
|
41
|
+
----
|
42
|
+
require "sidekiq/throttled"
|
43
|
+
----
|
44
|
+
|
45
|
+
Once you've done that you can include `Sidekiq::Throttled::Job` to your
|
46
|
+
job classes and configure throttling:
|
47
|
+
|
48
|
+
[source,ruby]
|
49
|
+
----
|
50
|
+
class MyJob
|
51
|
+
include Sidekiq::Job
|
52
|
+
include Sidekiq::Throttled::Job
|
53
|
+
|
54
|
+
sidekiq_options :queue => :my_queue
|
55
|
+
|
56
|
+
sidekiq_throttle(
|
57
|
+
# Allow maximum 10 concurrent jobs of this class at a time.
|
58
|
+
concurrency: { limit: 10 },
|
59
|
+
# Allow maximum 1K jobs being processed within one hour window.
|
60
|
+
threshold: { limit: 1_000, period: 1.hour }
|
61
|
+
)
|
62
|
+
|
63
|
+
def perform
|
64
|
+
# ...
|
65
|
+
end
|
66
|
+
end
|
67
|
+
----
|
68
|
+
|
69
|
+
TIP: `Sidekiq::Throttled::Job` is aliased as `Sidekiq::Throttled::Worker`,
|
70
|
+
thus if you're using `Sidekiq::Worker` naming convention, you can use the
|
71
|
+
alias for consistency:
|
72
|
+
|
73
|
+
[source,ruby]
|
74
|
+
----
|
75
|
+
class MyWorker
|
76
|
+
include Sidekiq::Worker
|
77
|
+
include Sidekiq::Throttled::Worker
|
78
|
+
|
79
|
+
# ...
|
80
|
+
end
|
81
|
+
----
|
82
|
+
|
83
|
+
|
84
|
+
=== Configuration
|
85
|
+
|
86
|
+
[source,ruby]
|
87
|
+
----
|
88
|
+
Sidekiq::Throttled.configure do |config|
|
89
|
+
# Period in seconds to exclude queue from polling in case it returned
|
90
|
+
# {config.cooldown_threshold} amount of throttled jobs in a row. Set
|
91
|
+
# this value to `nil` to disable cooldown manager completely.
|
92
|
+
# Default: 2.0
|
93
|
+
config.cooldown_period = 2.0
|
94
|
+
|
95
|
+
# Exclude queue from polling after it returned given amount of throttled
|
96
|
+
# jobs in a row.
|
97
|
+
# Default: 1 (cooldown after first throttled job)
|
98
|
+
config.cooldown_threshold = 1
|
99
|
+
end
|
100
|
+
----
|
101
|
+
|
102
|
+
|
103
|
+
==== Middleware(s)
|
104
|
+
|
105
|
+
`Sidekiq::Throttled` relies on following bundled middlewares:
|
106
|
+
|
107
|
+
* `Sidekiq::Throttled::Middlewares::Server`
|
108
|
+
|
109
|
+
The middleware is automatically injected when you require `sidekiq/throttled`.
|
110
|
+
In rare cases, when this causes an issue, you can change middleware order manually:
|
111
|
+
|
112
|
+
[source,ruby]
|
113
|
+
----
|
114
|
+
Sidekiq.configure_server do |config|
|
115
|
+
# ...
|
116
|
+
|
117
|
+
config.server_middleware do |chain|
|
118
|
+
chain.prepend(Sidekiq::Throttled::Middlewares::Server)
|
119
|
+
end
|
120
|
+
end
|
121
|
+
----
|
122
|
+
|
123
|
+
See: https://github.com/sidekiq/sidekiq/blob/main/lib/sidekiq/middleware/chain.rb
|
124
|
+
|
125
|
+
|
126
|
+
=== Observer
|
127
|
+
|
128
|
+
You can specify an observer that will be called on throttling. To do so pass an
|
129
|
+
`:observer` option with callable object:
|
130
|
+
|
131
|
+
[source,ruby]
|
132
|
+
----
|
133
|
+
class MyJob
|
134
|
+
include Sidekiq::Job
|
135
|
+
include Sidekiq::Throttled::Job
|
136
|
+
|
137
|
+
MY_OBSERVER = lambda do |strategy, *args|
|
138
|
+
# do something
|
139
|
+
end
|
140
|
+
|
141
|
+
sidekiq_options queue: :my_queue
|
142
|
+
|
143
|
+
sidekiq_throttle(
|
144
|
+
concurrency: { limit: 10 },
|
145
|
+
threshold: { limit: 100, period: 1.hour },
|
146
|
+
observer: MY_OBSERVER
|
147
|
+
)
|
148
|
+
|
149
|
+
def perform(*args)
|
150
|
+
# ...
|
151
|
+
end
|
152
|
+
end
|
153
|
+
----
|
154
|
+
|
155
|
+
Observer will receive `strategy, *args` arguments, where `strategy` is a Symbol
|
156
|
+
`:concurrency` or `:threshold`, and `*args` are the arguments that were passed
|
157
|
+
to the job.
|
158
|
+
|
159
|
+
|
160
|
+
=== Dynamic throttling
|
161
|
+
|
162
|
+
You can throttle jobs dynamically with `:key_suffix` option:
|
163
|
+
|
164
|
+
[source,ruby]
|
165
|
+
----
|
166
|
+
class MyJob
|
167
|
+
include Sidekiq::Job
|
168
|
+
include Sidekiq::Throttled::Job
|
169
|
+
|
170
|
+
sidekiq_options queue: :my_queue
|
171
|
+
|
172
|
+
sidekiq_throttle(
|
173
|
+
# Allow maximum 10 concurrent jobs per user at a time.
|
174
|
+
concurrency: { limit: 10, key_suffix: -> (user_id) { user_id } }
|
175
|
+
)
|
176
|
+
|
177
|
+
def perform(user_id)
|
178
|
+
# ...
|
179
|
+
end
|
180
|
+
end
|
181
|
+
----
|
182
|
+
|
183
|
+
You can also supply dynamic values for limits and periods by supplying a proc
|
184
|
+
for these values. The proc will be evaluated at the time the job is fetched
|
185
|
+
and will receive the same arguments that are passed to the job.
|
186
|
+
|
187
|
+
[source,ruby]
|
188
|
+
----
|
189
|
+
class MyJob
|
190
|
+
include Sidekiq::Job
|
191
|
+
include Sidekiq::Throttled::Job
|
192
|
+
|
193
|
+
sidekiq_options queue: :my_queue
|
194
|
+
|
195
|
+
sidekiq_throttle(
|
196
|
+
# Allow maximum 1000 concurrent jobs of this class at a time for VIPs and 10 for all other users.
|
197
|
+
concurrency: {
|
198
|
+
limit: ->(user_id) { User.vip?(user_id) ? 1_000 : 10 },
|
199
|
+
key_suffix: ->(user_id) { User.vip?(user_id) ? "vip" : "std" }
|
200
|
+
},
|
201
|
+
# Allow 1000 jobs/hour to be processed for VIPs and 10/day for all others
|
202
|
+
threshold: {
|
203
|
+
limit: ->(user_id) { User.vip?(user_id) ? 1_000 : 10 },
|
204
|
+
period: ->(user_id) { User.vip?(user_id) ? 1.hour : 1.day },
|
205
|
+
key_suffix: ->(user_id) { User.vip?(user_id) ? "vip" : "std" }
|
206
|
+
}
|
207
|
+
)
|
208
|
+
|
209
|
+
def perform(user_id)
|
210
|
+
# ...
|
211
|
+
end
|
212
|
+
end
|
213
|
+
----
|
214
|
+
|
215
|
+
You also can use several different keys to throttle one worker.
|
216
|
+
|
217
|
+
[source,ruby]
|
218
|
+
----
|
219
|
+
class MyJob
|
220
|
+
include Sidekiq::Job
|
221
|
+
include Sidekiq::Throttled::Job
|
222
|
+
|
223
|
+
sidekiq_options queue: :my_queue
|
224
|
+
|
225
|
+
sidekiq_throttle(
|
226
|
+
# Allow maximum 10 concurrent jobs per project at a time and maximum 2 jobs per user
|
227
|
+
concurrency: [
|
228
|
+
{ limit: 10, key_suffix: -> (project_id, user_id) { project_id } },
|
229
|
+
{ limit: 2, key_suffix: -> (project_id, user_id) { user_id } }
|
230
|
+
]
|
231
|
+
# For :threshold it works the same
|
232
|
+
)
|
233
|
+
|
234
|
+
def perform(project_id, user_id)
|
235
|
+
# ...
|
236
|
+
end
|
237
|
+
end
|
238
|
+
----
|
239
|
+
|
240
|
+
IMPORTANT: Don't forget to specify `:key_suffix` and make it return different
|
241
|
+
values if you are using dynamic limit/period options. Otherwise, you risk
|
242
|
+
getting into some trouble.
|
243
|
+
|
244
|
+
|
245
|
+
=== Concurrency throttling fine-tuning
|
246
|
+
|
247
|
+
Concurrency throttling is based on distributed locks. Those locks have default
|
248
|
+
time to live (TTL) set to 15 minutes. If your job takes more than 15 minutes
|
249
|
+
to finish, lock will be released and you might end up with more jobs running
|
250
|
+
concurrently than you expect.
|
251
|
+
|
252
|
+
This is done to avoid deadlocks - when by any reason (e.g. Sidekiq process was
|
253
|
+
OOM-killed) cleanup middleware wasn't executed and locks were not released.
|
254
|
+
|
255
|
+
If your job takes more than 15 minutes to complete, you can tune concurrency
|
256
|
+
lock TTL to fit your needs:
|
257
|
+
|
258
|
+
[source,ruby]
|
259
|
+
----
|
260
|
+
# Set concurrency strategy lock TTL to 1 hour.
|
261
|
+
sidekiq_throttle(concurrency: { limit: 20, ttl: 1.hour.to_i })
|
262
|
+
----
|
263
|
+
|
264
|
+
|
265
|
+
== Supported Ruby Versions
|
266
|
+
|
267
|
+
This library aims to support and is tested against the following Ruby versions:
|
268
|
+
|
269
|
+
* Ruby 2.7.x
|
270
|
+
* Ruby 3.0.x
|
271
|
+
* Ruby 3.1.x
|
272
|
+
* Ruby 3.2.x
|
273
|
+
* Ruby 3.3.x
|
274
|
+
|
275
|
+
If something doesn't work on one of these versions, it's a bug.
|
276
|
+
|
277
|
+
This library may inadvertently work (or seem to work) on other Ruby versions,
|
278
|
+
however support will only be provided for the versions listed above.
|
279
|
+
|
280
|
+
If you would like this library to support another Ruby version or
|
281
|
+
implementation, you may volunteer to be a maintainer. Being a maintainer
|
282
|
+
entails making sure all tests run and pass on that implementation. When
|
283
|
+
something breaks on your implementation, you will be responsible for providing
|
284
|
+
patches in a timely fashion. If critical issues for a particular implementation
|
285
|
+
exist at the time of a major release, support for that Ruby version may be
|
286
|
+
dropped.
|
287
|
+
|
288
|
+
|
289
|
+
== Supported Sidekiq Versions
|
290
|
+
|
291
|
+
This library aims to support and work with following Sidekiq versions:
|
292
|
+
|
293
|
+
* Sidekiq 6.5.x
|
294
|
+
* Sidekiq 7.0.x
|
295
|
+
* Sidekiq 7.1.x
|
296
|
+
* Sidekiq 7.2.x
|
297
|
+
|
298
|
+
And the following Sidekiq Pro versions:
|
299
|
+
|
300
|
+
* Sidekiq Pro 7.0.x
|
301
|
+
* Sidekiq Pro 7.1.x
|
302
|
+
* Sidekiq Pro 7.2.x
|
303
|
+
|
304
|
+
== Development
|
305
|
+
|
306
|
+
bundle install
|
307
|
+
bundle exec appraisal generate
|
308
|
+
bundle exec appraisal install
|
309
|
+
bundle exec rake
|
310
|
+
|
311
|
+
=== Sidekiq-Pro
|
312
|
+
|
313
|
+
If you're working on Sidekiq-Pro support make sure that you have Sidekiq-Pro
|
314
|
+
license set either in the global config, or in `BUNDLE_GEMS\__CONTRIBSYS__COM`
|
315
|
+
environment variable.
|
316
|
+
|
317
|
+
== Contributing
|
318
|
+
|
319
|
+
* Fork sidekiq-throttled on GitHub
|
320
|
+
* Make your changes
|
321
|
+
* Ensure all tests pass (`bundle exec rake`)
|
322
|
+
* Send a pull request
|
323
|
+
* If we like them we'll merge them
|
324
|
+
* If we've accepted a patch, feel free to ask for commit access!
|
325
|
+
|
326
|
+
|
327
|
+
== Endorsement
|
328
|
+
|
329
|
+
https://github.com/sensortower[image:sensortower.svg[SensorTower]]
|
330
|
+
|
331
|
+
The initial work on the project was initiated to address the needs of
|
332
|
+
https://github.com/sensortower[SensorTower].
|
@@ -0,0 +1,44 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
module Throttled
|
5
|
+
# Configuration object.
|
6
|
+
class Config
|
7
|
+
# Period in seconds to exclude queue from polling in case it returned
|
8
|
+
# {#cooldown_threshold} amount of throttled jobs in a row.
|
9
|
+
#
|
10
|
+
# Set this to `nil` to disable cooldown completely.
|
11
|
+
#
|
12
|
+
# @return [Float, nil]
|
13
|
+
attr_reader :cooldown_period
|
14
|
+
|
15
|
+
# Amount of throttled jobs returned from the queue subsequently after
|
16
|
+
# which queue will be excluded from polling for the durations of
|
17
|
+
# {#cooldown_period}.
|
18
|
+
#
|
19
|
+
# @return [Integer]
|
20
|
+
attr_reader :cooldown_threshold
|
21
|
+
|
22
|
+
def initialize
|
23
|
+
@cooldown_period = 2.0
|
24
|
+
@cooldown_threshold = 1
|
25
|
+
end
|
26
|
+
|
27
|
+
# @!attribute [w] cooldown_period
|
28
|
+
def cooldown_period=(value)
|
29
|
+
raise TypeError, "unexpected type #{value.class}" unless value.nil? || value.is_a?(Float)
|
30
|
+
raise ArgumentError, "period must be positive" unless value.nil? || value.positive?
|
31
|
+
|
32
|
+
@cooldown_period = value
|
33
|
+
end
|
34
|
+
|
35
|
+
# @!attribute [w] cooldown_threshold
|
36
|
+
def cooldown_threshold=(value)
|
37
|
+
raise TypeError, "unexpected type #{value.class}" unless value.is_a?(Integer)
|
38
|
+
raise ArgumentError, "threshold must be positive" unless value.positive?
|
39
|
+
|
40
|
+
@cooldown_threshold = value
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
@@ -0,0 +1,55 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "concurrent"
|
4
|
+
|
5
|
+
require_relative "./expirable_set"
|
6
|
+
|
7
|
+
module Sidekiq
|
8
|
+
module Throttled
|
9
|
+
# @api internal
|
10
|
+
#
|
11
|
+
# Queues cooldown manager. Tracks list of queues that should be temporarily
|
12
|
+
# (for the duration of {Config#cooldown_period}) excluded from polling.
|
13
|
+
class Cooldown
|
14
|
+
class << self
|
15
|
+
# Returns new {Cooldown} instance if {Config#cooldown_period} is not `nil`.
|
16
|
+
#
|
17
|
+
# @param config [Config]
|
18
|
+
# @return [Cooldown, nil]
|
19
|
+
def [](config)
|
20
|
+
new(config) if config.cooldown_period
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
# @param config [Config]
|
25
|
+
def initialize(config)
|
26
|
+
@queues = ExpirableSet.new(config.cooldown_period)
|
27
|
+
@threshold = config.cooldown_threshold
|
28
|
+
@tracker = Concurrent::Map.new
|
29
|
+
end
|
30
|
+
|
31
|
+
# Notify that given queue returned job that was throttled.
|
32
|
+
#
|
33
|
+
# @param queue [String]
|
34
|
+
# @return [void]
|
35
|
+
def notify_throttled(queue)
|
36
|
+
@queues.add(queue) if @threshold <= @tracker.merge_pair(queue, 1, &:succ)
|
37
|
+
end
|
38
|
+
|
39
|
+
# Notify that given queue returned job that was not throttled.
|
40
|
+
#
|
41
|
+
# @param queue [String]
|
42
|
+
# @return [void]
|
43
|
+
def notify_admitted(queue)
|
44
|
+
@tracker.delete(queue)
|
45
|
+
end
|
46
|
+
|
47
|
+
# List of queues that should not be polled
|
48
|
+
#
|
49
|
+
# @return [Array<String>]
|
50
|
+
def queues
|
51
|
+
@queues.to_a
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "concurrent"
|
4
|
+
|
5
|
+
module Sidekiq
|
6
|
+
module Throttled
|
7
|
+
# @api internal
|
8
|
+
#
|
9
|
+
# Set of elements with expirations.
|
10
|
+
#
|
11
|
+
# @example
|
12
|
+
# set = ExpirableSet.new(10.0)
|
13
|
+
# set.add("a")
|
14
|
+
# sleep(5)
|
15
|
+
# set.add("b")
|
16
|
+
# set.to_a # => ["a", "b"]
|
17
|
+
# sleep(5)
|
18
|
+
# set.to_a # => ["b"]
|
19
|
+
class ExpirableSet
|
20
|
+
include Enumerable
|
21
|
+
|
22
|
+
# @param ttl [Float] expiration is seconds
|
23
|
+
# @raise [ArgumentError] if `ttl` is not positive Float
|
24
|
+
def initialize(ttl)
|
25
|
+
raise ArgumentError, "ttl must be positive Float" unless ttl.is_a?(Float) && ttl.positive?
|
26
|
+
|
27
|
+
@elements = Concurrent::Map.new
|
28
|
+
@ttl = ttl
|
29
|
+
end
|
30
|
+
|
31
|
+
# @param element [Object]
|
32
|
+
# @return [ExpirableSet] self
|
33
|
+
def add(element)
|
34
|
+
# cleanup expired elements to avoid mem-leak
|
35
|
+
horizon = now
|
36
|
+
expired = @elements.each_pair.select { |(_, sunset)| expired?(sunset, horizon) }
|
37
|
+
expired.each { |pair| @elements.delete_pair(*pair) }
|
38
|
+
|
39
|
+
# add new element
|
40
|
+
@elements[element] = now + @ttl
|
41
|
+
|
42
|
+
self
|
43
|
+
end
|
44
|
+
|
45
|
+
# @yield [Object] Gives each live (not expired) element to the block
|
46
|
+
def each
|
47
|
+
return to_enum __method__ unless block_given?
|
48
|
+
|
49
|
+
horizon = now
|
50
|
+
|
51
|
+
@elements.each_pair do |element, sunset|
|
52
|
+
yield element unless expired?(sunset, horizon)
|
53
|
+
end
|
54
|
+
|
55
|
+
self
|
56
|
+
end
|
57
|
+
|
58
|
+
private
|
59
|
+
|
60
|
+
# @return [Float]
|
61
|
+
def now
|
62
|
+
::Process.clock_gettime(::Process::CLOCK_MONOTONIC)
|
63
|
+
end
|
64
|
+
|
65
|
+
def expired?(sunset, horizon)
|
66
|
+
sunset <= horizon
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
@@ -1,7 +1,7 @@
|
|
1
1
|
# frozen_string_literal: true
|
2
2
|
|
3
3
|
# internal
|
4
|
-
|
4
|
+
require_relative "./registry"
|
5
5
|
|
6
6
|
module Sidekiq
|
7
7
|
module Throttled
|
@@ -13,7 +13,7 @@ module Sidekiq
|
|
13
13
|
# include Sidekiq::Job
|
14
14
|
# include Sidekiq::Throttled::Job
|
15
15
|
#
|
16
|
-
#
|
16
|
+
# sidekiq_options :queue => :my_queue
|
17
17
|
# sidekiq_throttle :threshold => { :limit => 123, :period => 1.hour }
|
18
18
|
#
|
19
19
|
# def perform
|
@@ -29,8 +29,8 @@ module Sidekiq
|
|
29
29
|
# in order to make API inline with `include Sidekiq::Job`.
|
30
30
|
#
|
31
31
|
# @private
|
32
|
-
def self.included(
|
33
|
-
|
32
|
+
def self.included(base)
|
33
|
+
base.extend(ClassMethods)
|
34
34
|
end
|
35
35
|
|
36
36
|
# Helper methods added to the singleton class of destination
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
module Throttled
|
5
|
+
class Message
|
6
|
+
def initialize(item)
|
7
|
+
@item = item.is_a?(Hash) ? item : parse(item)
|
8
|
+
end
|
9
|
+
|
10
|
+
def job_class
|
11
|
+
@item.fetch("wrapped") { @item["class"] }
|
12
|
+
end
|
13
|
+
|
14
|
+
def job_args
|
15
|
+
@item.key?("wrapped") ? @item.dig("args", 0, "arguments") : @item["args"]
|
16
|
+
end
|
17
|
+
|
18
|
+
def job_id
|
19
|
+
@item["jid"]
|
20
|
+
end
|
21
|
+
|
22
|
+
private
|
23
|
+
|
24
|
+
def parse(item)
|
25
|
+
item = Sidekiq.load_json(item)
|
26
|
+
item.is_a?(Hash) ? item : {}
|
27
|
+
rescue JSON::ParserError
|
28
|
+
{}
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,28 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# internal
|
4
|
+
require_relative "../message"
|
5
|
+
require_relative "../registry"
|
6
|
+
|
7
|
+
module Sidekiq
|
8
|
+
module Throttled
|
9
|
+
module Middlewares
|
10
|
+
# Server middleware required for Sidekiq::Throttled functioning.
|
11
|
+
class Server
|
12
|
+
include Sidekiq::ServerMiddleware
|
13
|
+
|
14
|
+
def call(_worker, msg, _queue)
|
15
|
+
yield
|
16
|
+
ensure
|
17
|
+
message = Message.new(msg)
|
18
|
+
|
19
|
+
if message.job_class && message.job_id
|
20
|
+
Registry.get(message.job_class) do |strategy|
|
21
|
+
strategy.finalize!(message.job_id, *message.job_args)
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
end
|
28
|
+
end
|
@@ -0,0 +1,45 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "sidekiq"
|
4
|
+
require "sidekiq/fetch"
|
5
|
+
|
6
|
+
require_relative "./throttled_retriever"
|
7
|
+
|
8
|
+
module Sidekiq
|
9
|
+
module Throttled
|
10
|
+
module Patches
|
11
|
+
module BasicFetch
|
12
|
+
def self.prepended(base)
|
13
|
+
base.prepend(ThrottledRetriever)
|
14
|
+
end
|
15
|
+
|
16
|
+
private
|
17
|
+
|
18
|
+
# Pushes job back to the head of the queue, so that job won't be tried
|
19
|
+
# immediately after it was requeued (in most cases).
|
20
|
+
#
|
21
|
+
# @note This is triggered when job is throttled. So it is same operation
|
22
|
+
# Sidekiq performs upon `Sidekiq::Worker.perform_async` call.
|
23
|
+
#
|
24
|
+
# @return [void]
|
25
|
+
def requeue_throttled(work)
|
26
|
+
redis { |conn| conn.lpush(work.queue, work.job) }
|
27
|
+
end
|
28
|
+
|
29
|
+
# Returns list of queues to try to fetch jobs from.
|
30
|
+
#
|
31
|
+
# @note It may return an empty array.
|
32
|
+
# @param [Array<String>] queues
|
33
|
+
# @return [Array<String>]
|
34
|
+
def queues_cmd
|
35
|
+
throttled_queues = Throttled.cooldown&.queues
|
36
|
+
return super if throttled_queues.nil? || throttled_queues.empty?
|
37
|
+
|
38
|
+
super - throttled_queues
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
Sidekiq::BasicFetch.prepend(Sidekiq::Throttled::Patches::BasicFetch)
|
@@ -0,0 +1,52 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require "sidekiq"
|
4
|
+
|
5
|
+
require_relative "./throttled_retriever"
|
6
|
+
|
7
|
+
module Sidekiq
|
8
|
+
module Throttled
|
9
|
+
module Patches
|
10
|
+
module SuperFetch
|
11
|
+
def self.prepended(base)
|
12
|
+
base.prepend(ThrottledRetriever)
|
13
|
+
end
|
14
|
+
|
15
|
+
private
|
16
|
+
|
17
|
+
# Calls SuperFetch UnitOfWork's requeue to remove the job from the
|
18
|
+
# temporary queue and push job back to the head of the queue, so that
|
19
|
+
# the job won't be tried immediately after it was requeued (in most cases).
|
20
|
+
#
|
21
|
+
# @note This is triggered when job is throttled.
|
22
|
+
#
|
23
|
+
# @return [void]
|
24
|
+
def requeue_throttled(work)
|
25
|
+
# SuperFetch UnitOfWork's requeue will remove it from the temporary
|
26
|
+
# queue and then requeue it, so no acknowledgement call is needed.
|
27
|
+
work.requeue
|
28
|
+
end
|
29
|
+
|
30
|
+
# Returns list of non-paused queues to try to fetch jobs from.
|
31
|
+
#
|
32
|
+
# @note It may return an empty array.
|
33
|
+
# @return [Array<Array(String, String)>]
|
34
|
+
def active_queues
|
35
|
+
# Create a hash of throttled queues for fast lookup
|
36
|
+
throttled_queues = Throttled.cooldown&.queues&.to_h { |queue| [queue, true] }
|
37
|
+
return super if throttled_queues.nil? || throttled_queues.empty?
|
38
|
+
|
39
|
+
# Reject throttled queues from the list of active queues
|
40
|
+
super.reject { |queue, _private_queue| throttled_queues[queue] }
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
begin
|
48
|
+
require "sidekiq/pro/super_fetch"
|
49
|
+
Sidekiq::Pro::SuperFetch.prepend(Sidekiq::Throttled::Patches::SuperFetch)
|
50
|
+
rescue LoadError
|
51
|
+
# Sidekiq Pro is not available
|
52
|
+
end
|