sidekiq-throttled 0.13.0 → 0.14.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +2 -4
- data/.rubocop/layout.yml +24 -0
- data/.rubocop/lint.yml +41 -0
- data/.rubocop/metrics.yml +4 -0
- data/.rubocop/performance.yml +25 -0
- data/.rubocop/rspec.yml +3 -0
- data/.rubocop/style.yml +84 -0
- data/.rubocop.yml +13 -94
- data/.rubocop_todo.yml +10 -35
- data/Gemfile +3 -3
- data/README.md +33 -10
- data/lib/sidekiq/throttled/communicator/callbacks.rb +6 -6
- data/lib/sidekiq/throttled/communicator/listener.rb +3 -3
- data/lib/sidekiq/throttled/queue_name.rb +1 -1
- data/lib/sidekiq/throttled/registry.rb +0 -5
- data/lib/sidekiq/throttled/strategy.rb +15 -17
- data/lib/sidekiq/throttled/strategy_collection.rb +70 -0
- data/lib/sidekiq/throttled/utils.rb +1 -1
- data/lib/sidekiq/throttled/version.rb +1 -1
- data/lib/sidekiq/throttled/web/summary_fix.rb +1 -1
- data/lib/sidekiq/throttled/web/throttled.html.erb +6 -2
- data/lib/sidekiq/throttled/web.rb +1 -1
- data/lib/sidekiq/throttled/worker.rb +2 -2
- data/sidekiq-throttled.gemspec +2 -0
- metadata +15 -8
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: fe88fba2840a8a649b6a02951a246176e8b548257997aeaf922c7582e8c27e17
|
4
|
+
data.tar.gz: 76ba37d70509e33a266e7ad1e9e51babd4afd874788a1548b1aabc3a1c1cad87
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 14979e92d96a56096f19e61a40f8343025de2ccb4a57ede6f311be83acd80cca44d9bae7448ad2453e9539aa599c6154d2eec9f2a285148f14d2ad70fa9d86c4
|
7
|
+
data.tar.gz: 8ad7ee2999f8ab2645bdff54efc2b2592b5d416807795f58454af1a5f03ae3099975db492bbab4658cd2b9d5160af705c5509536c9dd7dddaa1db35c5ebe4448
|
data/.github/workflows/ci.yml
CHANGED
@@ -12,10 +12,8 @@ jobs:
|
|
12
12
|
|
13
13
|
strategy:
|
14
14
|
matrix:
|
15
|
-
ruby: [ "2.
|
16
|
-
sidekiq: [ "5.0", "5.1", "5.2", "6.0" ]
|
17
|
-
exclude:
|
18
|
-
- { ruby: "2.4", sidekiq: "6.0" }
|
15
|
+
ruby: [ "2.6", "2.7" ]
|
16
|
+
sidekiq: [ "5.0", "5.1", "5.2", "6.0", "6.1" ]
|
19
17
|
|
20
18
|
runs-on: ubuntu-latest
|
21
19
|
|
data/.rubocop/layout.yml
ADDED
@@ -0,0 +1,24 @@
|
|
1
|
+
Layout/ArgumentAlignment:
|
2
|
+
EnforcedStyle: with_fixed_indentation
|
3
|
+
|
4
|
+
Layout/EmptyLinesAroundAttributeAccessor:
|
5
|
+
Enabled: true
|
6
|
+
|
7
|
+
Layout/FirstArrayElementIndentation:
|
8
|
+
EnforcedStyle: consistent
|
9
|
+
|
10
|
+
Layout/FirstHashElementIndentation:
|
11
|
+
Enabled: true
|
12
|
+
EnforcedStyle: consistent
|
13
|
+
|
14
|
+
Layout/HashAlignment:
|
15
|
+
Enabled: true
|
16
|
+
EnforcedHashRocketStyle: table
|
17
|
+
EnforcedColonStyle: table
|
18
|
+
|
19
|
+
Layout/SpaceAroundMethodCallOperator:
|
20
|
+
Enabled: true
|
21
|
+
|
22
|
+
Layout/MultilineMethodCallIndentation:
|
23
|
+
Enabled: true
|
24
|
+
EnforcedStyle: indented
|
data/.rubocop/lint.yml
ADDED
@@ -0,0 +1,41 @@
|
|
1
|
+
Lint/BinaryOperatorWithIdenticalOperands:
|
2
|
+
Enabled: true
|
3
|
+
|
4
|
+
Lint/DeprecatedOpenSSLConstant:
|
5
|
+
Enabled: true
|
6
|
+
|
7
|
+
Lint/DuplicateElsifCondition:
|
8
|
+
Enabled: true
|
9
|
+
|
10
|
+
Lint/DuplicateRescueException:
|
11
|
+
Enabled: true
|
12
|
+
|
13
|
+
Lint/EmptyConditionalBody:
|
14
|
+
Enabled: true
|
15
|
+
|
16
|
+
Lint/FloatComparison:
|
17
|
+
Enabled: true
|
18
|
+
|
19
|
+
Lint/MissingSuper:
|
20
|
+
Enabled: true
|
21
|
+
|
22
|
+
Lint/MixedRegexpCaptureTypes:
|
23
|
+
Enabled: true
|
24
|
+
|
25
|
+
Lint/OutOfRangeRegexpRef:
|
26
|
+
Enabled: true
|
27
|
+
|
28
|
+
Lint/RaiseException:
|
29
|
+
Enabled: true
|
30
|
+
|
31
|
+
Lint/SelfAssignment:
|
32
|
+
Enabled: true
|
33
|
+
|
34
|
+
Lint/StructNewOverride:
|
35
|
+
Enabled: true
|
36
|
+
|
37
|
+
Lint/TopLevelReturnWithArgument:
|
38
|
+
Enabled: true
|
39
|
+
|
40
|
+
Lint/UnreachableLoop:
|
41
|
+
Enabled: true
|
@@ -0,0 +1,25 @@
|
|
1
|
+
Performance/AncestorsInclude:
|
2
|
+
Enabled: true
|
3
|
+
|
4
|
+
Performance/BigDecimalWithNumericArgument:
|
5
|
+
Enabled: true
|
6
|
+
|
7
|
+
Performance/RedundantSortBlock:
|
8
|
+
Enabled: true
|
9
|
+
|
10
|
+
Performance/RedundantStringChars:
|
11
|
+
Enabled: true
|
12
|
+
|
13
|
+
Performance/ReverseFirst:
|
14
|
+
Enabled: true
|
15
|
+
|
16
|
+
Performance/SortReverse:
|
17
|
+
Enabled: true
|
18
|
+
|
19
|
+
Performance/Squeeze:
|
20
|
+
Enabled: true
|
21
|
+
|
22
|
+
Performance/StringInclude:
|
23
|
+
Enabled: true
|
24
|
+
|
25
|
+
|
data/.rubocop/rspec.yml
ADDED
data/.rubocop/style.yml
ADDED
@@ -0,0 +1,84 @@
|
|
1
|
+
Style/AccessorGrouping:
|
2
|
+
Enabled: true
|
3
|
+
|
4
|
+
Style/ArrayCoercion:
|
5
|
+
Enabled: true
|
6
|
+
|
7
|
+
Style/BisectedAttrAccessor:
|
8
|
+
Enabled: true
|
9
|
+
|
10
|
+
Style/CaseLikeIf:
|
11
|
+
Enabled: true
|
12
|
+
|
13
|
+
Style/Documentation:
|
14
|
+
Enabled: false
|
15
|
+
|
16
|
+
Style/ExplicitBlockArgument:
|
17
|
+
Enabled: true
|
18
|
+
|
19
|
+
Style/ExponentialNotation:
|
20
|
+
Enabled: true
|
21
|
+
|
22
|
+
Style/GlobalStdStream:
|
23
|
+
Enabled: true
|
24
|
+
|
25
|
+
Style/HashAsLastArrayItem:
|
26
|
+
Enabled: true
|
27
|
+
|
28
|
+
Style/HashEachMethods:
|
29
|
+
Enabled: true
|
30
|
+
|
31
|
+
Style/HashLikeCase:
|
32
|
+
Enabled: true
|
33
|
+
|
34
|
+
Style/HashSyntax:
|
35
|
+
Enabled: true
|
36
|
+
EnforcedStyle: hash_rockets
|
37
|
+
|
38
|
+
Style/HashTransformKeys:
|
39
|
+
Enabled: true
|
40
|
+
|
41
|
+
Style/HashTransformValues:
|
42
|
+
Enabled: true
|
43
|
+
|
44
|
+
Style/OptionalBooleanParameter:
|
45
|
+
Enabled: true
|
46
|
+
|
47
|
+
Style/RedundantAssignment:
|
48
|
+
Enabled: true
|
49
|
+
|
50
|
+
Style/RedundantFetchBlock:
|
51
|
+
Enabled: true
|
52
|
+
|
53
|
+
Style/RedundantFileExtensionInRequire:
|
54
|
+
Enabled: true
|
55
|
+
|
56
|
+
Style/RedundantRegexpCharacterClass:
|
57
|
+
Enabled: true
|
58
|
+
|
59
|
+
Style/RedundantRegexpEscape:
|
60
|
+
Enabled: true
|
61
|
+
|
62
|
+
Style/RegexpLiteral:
|
63
|
+
Enabled: true
|
64
|
+
EnforcedStyle: percent_r
|
65
|
+
|
66
|
+
Style/RescueStandardError:
|
67
|
+
Enabled: true
|
68
|
+
EnforcedStyle: implicit
|
69
|
+
|
70
|
+
Style/SingleArgumentDig:
|
71
|
+
Enabled: true
|
72
|
+
|
73
|
+
Style/SlicingWithRange:
|
74
|
+
Enabled: true
|
75
|
+
|
76
|
+
Style/StringConcatenation:
|
77
|
+
Enabled: true
|
78
|
+
|
79
|
+
Style/StringLiterals:
|
80
|
+
Enabled: true
|
81
|
+
EnforcedStyle: double_quotes
|
82
|
+
|
83
|
+
Style/YodaCondition:
|
84
|
+
Enabled: false
|
data/.rubocop.yml
CHANGED
@@ -1,100 +1,19 @@
|
|
1
|
-
inherit_from:
|
2
|
-
- .rubocop_todo.yml
|
3
|
-
|
4
1
|
require:
|
2
|
+
- rubocop-performance
|
5
3
|
- rubocop-rspec
|
6
4
|
|
7
|
-
|
5
|
+
inherit_from:
|
6
|
+
- .rubocop_todo.yml
|
7
|
+
- .rubocop/layout.yml
|
8
|
+
- .rubocop/lint.yml
|
9
|
+
- .rubocop/metrics.yml
|
10
|
+
- .rubocop/performance.yml
|
11
|
+
- .rubocop/rspec.yml
|
12
|
+
- .rubocop/style.yml
|
8
13
|
|
9
14
|
AllCops:
|
10
|
-
DisplayCopNames: true
|
11
|
-
TargetRubyVersion: 2.4
|
12
15
|
Exclude:
|
13
|
-
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
Layout/ArgumentAlignment:
|
18
|
-
EnforcedStyle: with_fixed_indentation
|
19
|
-
|
20
|
-
Layout/FirstArrayElementIndentation:
|
21
|
-
EnforcedStyle: consistent
|
22
|
-
|
23
|
-
Layout/FirstHashElementIndentation:
|
24
|
-
EnforcedStyle: consistent
|
25
|
-
|
26
|
-
Layout/HashAlignment:
|
27
|
-
EnforcedHashRocketStyle: table
|
28
|
-
|
29
|
-
Layout/LineLength:
|
30
|
-
Max: 100
|
31
|
-
|
32
|
-
Layout/MultilineMethodCallIndentation:
|
33
|
-
EnforcedStyle: indented
|
34
|
-
|
35
|
-
Layout/ParameterAlignment:
|
36
|
-
EnforcedStyle: with_fixed_indentation
|
37
|
-
|
38
|
-
Layout/SpaceAroundMethodCallOperator:
|
39
|
-
Enabled: true
|
40
|
-
|
41
|
-
Layout/SpaceInLambdaLiteral:
|
42
|
-
EnforcedStyle: require_space
|
43
|
-
|
44
|
-
## Lint ########################################################################
|
45
|
-
|
46
|
-
Lint/RaiseException:
|
47
|
-
Enabled: true
|
48
|
-
|
49
|
-
Lint/StructNewOverride:
|
50
|
-
Enabled: true
|
51
|
-
|
52
|
-
## Metrics #####################################################################
|
53
|
-
|
54
|
-
Metrics/BlockLength:
|
55
|
-
Exclude:
|
56
|
-
- "Guardfile"
|
57
|
-
- "spec/**/*"
|
58
|
-
|
59
|
-
## Styles ######################################################################
|
60
|
-
|
61
|
-
Style/Documentation:
|
62
|
-
Enabled: false
|
63
|
-
|
64
|
-
Style/ExponentialNotation:
|
65
|
-
Enabled: true
|
66
|
-
|
67
|
-
Style/HashEachMethods:
|
68
|
-
Enabled: true
|
69
|
-
|
70
|
-
Style/HashSyntax:
|
71
|
-
EnforcedStyle: hash_rockets
|
72
|
-
|
73
|
-
Style/HashTransformKeys:
|
74
|
-
Enabled: true
|
75
|
-
|
76
|
-
Style/HashTransformValues:
|
77
|
-
Enabled: true
|
78
|
-
|
79
|
-
# Follow your heart where it makes sense to use lambda or lambda literal.
|
80
|
-
# Enforcing it makes some pieces of code look REALLY terrible, e.g. in
|
81
|
-
# case of empty (noop) lambdas: `lambda { |_| }`.
|
82
|
-
Style/Lambda:
|
83
|
-
Enabled: false
|
84
|
-
|
85
|
-
# Enabling this cop makes Guardfile (which is full of pathname regexps)
|
86
|
-
# look absolutley style-inconsistent and terrible. In any case, this should
|
87
|
-
# be on developer's choice whenever to use `%r` or not. Just like we don't
|
88
|
-
# enforce to use `["foo"]` over `%w(foo)` and so on.
|
89
|
-
Style/RegexpLiteral:
|
90
|
-
Enabled: false
|
91
|
-
|
92
|
-
Style/RescueStandardError:
|
93
|
-
EnforcedStyle: implicit
|
94
|
-
|
95
|
-
Style/StringLiterals:
|
96
|
-
EnforcedStyle: double_quotes
|
97
|
-
|
98
|
-
# I prefer Yoda style instead, but there's no such enforcement style.
|
99
|
-
Style/YodaCondition:
|
100
|
-
Enabled: false
|
16
|
+
- gemfiles/**/*
|
17
|
+
- vendor/**/*
|
18
|
+
NewCops: enable
|
19
|
+
TargetRubyVersion: 2.4
|
data/.rubocop_todo.yml
CHANGED
@@ -1,57 +1,32 @@
|
|
1
1
|
# This configuration was generated by
|
2
2
|
# `rubocop --auto-gen-config`
|
3
|
-
# on
|
3
|
+
# on 2020-09-14 13:53:26 UTC using RuboCop version 0.90.0.
|
4
4
|
# The point is for the user to remove these configuration records
|
5
5
|
# one by one as the offenses are removed from the code base.
|
6
6
|
# Note that changes in the inspected code, or installation of new
|
7
7
|
# versions of RuboCop, may require this file to be generated again.
|
8
8
|
|
9
|
-
# Offense count:
|
10
|
-
# Configuration parameters: CountComments.
|
11
|
-
Metrics/MethodLength:
|
12
|
-
Max: 13
|
13
|
-
|
14
|
-
# Offense count: 25
|
15
|
-
# Configuration parameters: Max.
|
16
|
-
RSpec/ExampleLength:
|
17
|
-
Exclude:
|
18
|
-
- 'spec/sidekiq/throttled/communicator/callbacks_spec.rb'
|
19
|
-
- 'spec/sidekiq/throttled/communicator/listener_spec.rb'
|
20
|
-
- 'spec/sidekiq/throttled/communicator_spec.rb'
|
21
|
-
- 'spec/sidekiq/throttled/fetch_spec.rb'
|
22
|
-
- 'spec/sidekiq/throttled/queues_pauser_spec.rb'
|
23
|
-
- 'spec/sidekiq/throttled/strategy_spec.rb'
|
24
|
-
- 'spec/sidekiq/throttled/web/queues_spec.rb'
|
25
|
-
- 'spec/sidekiq/throttled_spec.rb'
|
26
|
-
|
27
|
-
# Offense count: 62
|
9
|
+
# Offense count: 68
|
28
10
|
# Configuration parameters: .
|
29
11
|
# SupportedStyles: have_received, receive
|
30
12
|
RSpec/MessageSpies:
|
31
13
|
EnforcedStyle: receive
|
32
14
|
|
33
|
-
# Offense count:
|
34
|
-
# Configuration parameters: AggregateFailuresByDefault.
|
15
|
+
# Offense count: 22
|
35
16
|
RSpec/MultipleExpectations:
|
36
17
|
Max: 4
|
37
18
|
|
38
|
-
# Offense count:
|
19
|
+
# Offense count: 7
|
20
|
+
# Configuration parameters: AllowSubject.
|
21
|
+
RSpec/MultipleMemoizedHelpers:
|
22
|
+
Max: 6
|
23
|
+
|
24
|
+
# Offense count: 46
|
39
25
|
RSpec/NestedGroups:
|
40
26
|
Max: 5
|
41
27
|
|
42
|
-
# Offense count:
|
28
|
+
# Offense count: 4
|
43
29
|
RSpec/SubjectStub:
|
44
30
|
Exclude:
|
45
31
|
- 'spec/sidekiq/throttled/communicator_spec.rb'
|
46
32
|
- 'spec/sidekiq/throttled/queues_pauser_spec.rb'
|
47
|
-
|
48
|
-
# Offense count: 1
|
49
|
-
# Configuration parameters: IgnoreSymbolicNames.
|
50
|
-
RSpec/VerifiedDoubles:
|
51
|
-
Exclude:
|
52
|
-
- 'spec/sidekiq/throttled/registry_spec.rb'
|
53
|
-
|
54
|
-
# Offense count: 1
|
55
|
-
Security/MarshalLoad:
|
56
|
-
Exclude:
|
57
|
-
- 'lib/sidekiq/throttled/communicator/listener.rb'
|
data/Gemfile
CHANGED
@@ -5,9 +5,9 @@ source "https://rubygems.org"
|
|
5
5
|
gem "appraisal"
|
6
6
|
gem "rake"
|
7
7
|
gem "rspec"
|
8
|
-
gem "rubocop", "~> 0.
|
9
|
-
gem "rubocop-performance", "~>1.
|
10
|
-
gem "rubocop-rspec", "~> 1.
|
8
|
+
gem "rubocop", "~> 0.90.0", :require => false
|
9
|
+
gem "rubocop-performance", "~> 1.8.0", :require => false
|
10
|
+
gem "rubocop-rspec", "~> 1.43.2", :require => false
|
11
11
|
gem "sidekiq"
|
12
12
|
|
13
13
|
group :development do
|
data/README.md
CHANGED
@@ -49,12 +49,12 @@ class MyWorker
|
|
49
49
|
|
50
50
|
sidekiq_options :queue => :my_queue
|
51
51
|
|
52
|
-
sidekiq_throttle(
|
52
|
+
sidekiq_throttle(
|
53
53
|
# Allow maximum 10 concurrent jobs of this class at a time.
|
54
54
|
:concurrency => { :limit => 10 },
|
55
55
|
# Allow maximum 1K jobs being processed within one hour window.
|
56
56
|
:threshold => { :limit => 1_000, :period => 1.hour }
|
57
|
-
|
57
|
+
)
|
58
58
|
|
59
59
|
def perform
|
60
60
|
# ...
|
@@ -78,11 +78,11 @@ class MyWorker
|
|
78
78
|
|
79
79
|
sidekiq_options :queue => :my_queue
|
80
80
|
|
81
|
-
sidekiq_throttle(
|
81
|
+
sidekiq_throttle(
|
82
82
|
:concurrency => { :limit => 10 },
|
83
83
|
:threshold => { :limit => 100, :period => 1.hour }
|
84
84
|
:observer => MY_OBSERVER
|
85
|
-
|
85
|
+
)
|
86
86
|
|
87
87
|
def perform(*args)
|
88
88
|
# ...
|
@@ -106,10 +106,10 @@ class MyWorker
|
|
106
106
|
|
107
107
|
sidekiq_options :queue => :my_queue
|
108
108
|
|
109
|
-
sidekiq_throttle(
|
109
|
+
sidekiq_throttle(
|
110
110
|
# Allow maximum 10 concurrent jobs per user at a time.
|
111
111
|
:concurrency => { :limit => 10, :key_suffix => -> (user_id) { user_id } }
|
112
|
-
|
112
|
+
)
|
113
113
|
|
114
114
|
def perform(user_id)
|
115
115
|
# ...
|
@@ -128,7 +128,7 @@ class MyWorker
|
|
128
128
|
|
129
129
|
sidekiq_options :queue => :my_queue
|
130
130
|
|
131
|
-
sidekiq_throttle(
|
131
|
+
sidekiq_throttle(
|
132
132
|
# Allow maximum 1000 concurrent jobs of this class at a time for VIPs and 10 for all other users.
|
133
133
|
:concurrency => {
|
134
134
|
:limit => ->(user_id) { User.vip?(user_id) ? 1_000 : 10 },
|
@@ -139,7 +139,7 @@ class MyWorker
|
|
139
139
|
:limit => ->(user_id) { User.vip?(user_id) ? 1_000 : 10 },
|
140
140
|
:period => ->(user_id) { User.vip?(user_id) ? 1.hour : 1.day },
|
141
141
|
:key_suffix => ->(user_id) { User.vip?(user_id) ? "vip" : "std" }
|
142
|
-
|
142
|
+
)
|
143
143
|
|
144
144
|
def perform(user_id)
|
145
145
|
# ...
|
@@ -147,6 +147,30 @@ class MyWorker
|
|
147
147
|
end
|
148
148
|
```
|
149
149
|
|
150
|
+
You also can use several different keys to throttle one worker.
|
151
|
+
|
152
|
+
``` ruby
|
153
|
+
class MyWorker
|
154
|
+
include Sidekiq::Worker
|
155
|
+
include Sidekiq::Throttled::Worker
|
156
|
+
|
157
|
+
sidekiq_options :queue => :my_queue
|
158
|
+
|
159
|
+
sidekiq_throttle(
|
160
|
+
# Allow maximum 10 concurrent jobs per project at a time and maximum 2 jobs per user
|
161
|
+
:concurrency => [
|
162
|
+
{ :limit => 10, :key_suffix => -> (project_id, user_id) { project_id } },
|
163
|
+
{ :limit => 2, :key_suffix => -> (project_id, user_id) { user_id } }
|
164
|
+
]
|
165
|
+
# For :threshold it works the same
|
166
|
+
)
|
167
|
+
|
168
|
+
def perform(project_id, user_id)
|
169
|
+
# ...
|
170
|
+
end
|
171
|
+
end
|
172
|
+
```
|
173
|
+
|
150
174
|
**NB** Don't forget to specify `:key_suffix` and make it return different values
|
151
175
|
if you are using dynamic limit/period options. Otherwise you risk getting into
|
152
176
|
some trouble.
|
@@ -209,9 +233,8 @@ end
|
|
209
233
|
This library aims to support and is [tested against][travis] the following Ruby
|
210
234
|
versions:
|
211
235
|
|
212
|
-
* Ruby 2.4.x
|
213
|
-
* Ruby 2.5.x
|
214
236
|
* Ruby 2.6.x
|
237
|
+
* Ruby 2.7.x
|
215
238
|
|
216
239
|
If something doesn't work on one of these versions, it's a bug.
|
217
240
|
|
@@ -53,19 +53,19 @@ module Sidekiq
|
|
53
53
|
# @param [#to_s] event
|
54
54
|
# @param [Object] payload
|
55
55
|
# @return [void]
|
56
|
-
def run(event, payload = nil)
|
56
|
+
def run(event, payload = nil) # rubocop:disable Metrics/MethodLength
|
57
57
|
@mutex.synchronize do
|
58
|
-
Fiber.new do
|
58
|
+
fiber = Fiber.new do
|
59
59
|
@handlers[event.to_s].each do |callback|
|
60
60
|
begin
|
61
61
|
callback.call(payload)
|
62
62
|
rescue => e
|
63
|
-
handle_exception(e,
|
64
|
-
:context => "sidekiq:throttled"
|
65
|
-
})
|
63
|
+
handle_exception(e, :context => "sidekiq:throttled")
|
66
64
|
end
|
67
65
|
end
|
68
|
-
end
|
66
|
+
end
|
67
|
+
|
68
|
+
fiber.resume
|
69
69
|
end
|
70
70
|
end
|
71
71
|
end
|
@@ -62,7 +62,7 @@ module Sidekiq
|
|
62
62
|
# - `Exception` is recorded to the log and re-raised.
|
63
63
|
#
|
64
64
|
# @return [void]
|
65
|
-
def listen
|
65
|
+
def listen # rubocop:disable Metrics/MethodLength
|
66
66
|
subscribe
|
67
67
|
rescue Sidekiq::Shutdown
|
68
68
|
@terminated = true
|
@@ -88,7 +88,7 @@ module Sidekiq
|
|
88
88
|
# @see http://redis.io/commands/subscribe
|
89
89
|
# @see Callbacks#run
|
90
90
|
# @return [void]
|
91
|
-
def subscribe
|
91
|
+
def subscribe # rubocop:disable Metrics/MethodLength
|
92
92
|
Sidekiq.redis do |conn|
|
93
93
|
conn.subscribe @channel do |on|
|
94
94
|
on.subscribe do
|
@@ -97,7 +97,7 @@ module Sidekiq
|
|
97
97
|
end
|
98
98
|
|
99
99
|
on.message do |_channel, data|
|
100
|
-
message, payload = Marshal.load(data)
|
100
|
+
message, payload = Marshal.load(data) # rubocop:disable Security/MarshalLoad:
|
101
101
|
@callbacks.run("message:#{message}", payload)
|
102
102
|
end
|
103
103
|
end
|
@@ -7,7 +7,7 @@ module Sidekiq
|
|
7
7
|
# @private
|
8
8
|
module QueueName
|
9
9
|
# RegExp used to stip out any redisr-namespace prefixes with `queue:`.
|
10
|
-
QUEUE_NAME_PREFIX_RE =
|
10
|
+
QUEUE_NAME_PREFIX_RE = %r{.*queue:}.freeze
|
11
11
|
private_constant :QUEUE_NAME_PREFIX_RE
|
12
12
|
|
13
13
|
class << self
|
@@ -18,20 +18,16 @@ module Sidekiq
|
|
18
18
|
|
19
19
|
# Adds strategy to the registry.
|
20
20
|
#
|
21
|
-
# @note prints a warning to STDERR upon duplicate strategy name
|
22
21
|
# @param (see Strategy#initialize)
|
23
22
|
# @return [Strategy]
|
24
23
|
def add(name, **kwargs)
|
25
24
|
name = name.to_s
|
26
25
|
|
27
|
-
warn "Duplicate strategy name: #{name}" if @strategies[name]
|
28
|
-
|
29
26
|
@strategies[name] = Strategy.new(name, **kwargs)
|
30
27
|
end
|
31
28
|
|
32
29
|
# Adds alias for existing strategy.
|
33
30
|
#
|
34
|
-
# @note prints a warning to STDERR upon duplicate strategy name
|
35
31
|
# @param (#to_s) new_name
|
36
32
|
# @param (#to_s) old_name
|
37
33
|
# @raise [RuntimeError] if no strategy found with `old_name`
|
@@ -40,7 +36,6 @@ module Sidekiq
|
|
40
36
|
new_name = new_name.to_s
|
41
37
|
old_name = old_name.to_s
|
42
38
|
|
43
|
-
warn "Duplicate strategy name: #{new_name}" if @strategies[new_name]
|
44
39
|
raise "Strategy not found: #{old_name}" unless @strategies[old_name]
|
45
40
|
|
46
41
|
@aliases[new_name] = @strategies[old_name]
|
@@ -2,6 +2,7 @@
|
|
2
2
|
|
3
3
|
# internal
|
4
4
|
require "sidekiq/throttled/errors"
|
5
|
+
require "sidekiq/throttled/strategy_collection"
|
5
6
|
require "sidekiq/throttled/strategy/concurrency"
|
6
7
|
require "sidekiq/throttled/strategy/threshold"
|
7
8
|
|
@@ -30,15 +31,20 @@ module Sidekiq
|
|
30
31
|
# See keyword args of {Strategy::Threshold#initialize} for details.
|
31
32
|
# @param [#call] key_suffix Dynamic key suffix generator.
|
32
33
|
# @param [#call] observer Process called after throttled.
|
33
|
-
def initialize(
|
34
|
-
|
35
|
-
concurrency: nil, threshold: nil, key_suffix: nil, observer: nil
|
36
|
-
)
|
37
|
-
@observer = observer
|
38
|
-
@concurrency = make_strategy(Concurrency, name, key_suffix, concurrency)
|
39
|
-
@threshold = make_strategy(Threshold, name, key_suffix, threshold)
|
34
|
+
def initialize(name, concurrency: nil, threshold: nil, key_suffix: nil, observer: nil) # rubocop:disable Metrics/MethodLength
|
35
|
+
@observer = observer
|
40
36
|
|
41
|
-
|
37
|
+
@concurrency = StrategyCollection.new(concurrency,
|
38
|
+
:strategy => Concurrency,
|
39
|
+
:name => name,
|
40
|
+
:key_suffix => key_suffix)
|
41
|
+
|
42
|
+
@threshold = StrategyCollection.new(threshold,
|
43
|
+
:strategy => Threshold,
|
44
|
+
:name => name,
|
45
|
+
:key_suffix => key_suffix)
|
46
|
+
|
47
|
+
return if @concurrency.any? || @threshold.any?
|
42
48
|
|
43
49
|
raise ArgumentError, "Neither :concurrency nor :threshold given"
|
44
50
|
end
|
@@ -60,6 +66,7 @@ module Sidekiq
|
|
60
66
|
|
61
67
|
if @threshold&.throttled?(*job_args)
|
62
68
|
@observer&.call(:threshold, *job_args)
|
69
|
+
|
63
70
|
finalize!(jid, *job_args)
|
64
71
|
return true
|
65
72
|
end
|
@@ -79,15 +86,6 @@ module Sidekiq
|
|
79
86
|
@concurrency&.reset!
|
80
87
|
@threshold&.reset!
|
81
88
|
end
|
82
|
-
|
83
|
-
private
|
84
|
-
|
85
|
-
# @return [Base, nil]
|
86
|
-
def make_strategy(strategy, name, key_suffix, options)
|
87
|
-
return unless options
|
88
|
-
|
89
|
-
strategy.new("throttled:#{name}", :key_suffix => key_suffix, **options)
|
90
|
-
end
|
91
89
|
end
|
92
90
|
end
|
93
91
|
end
|
@@ -0,0 +1,70 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
# internal
|
4
|
+
module Sidekiq
|
5
|
+
module Throttled
|
6
|
+
# Collection which transparently group several meta-strategies of one kind
|
7
|
+
#
|
8
|
+
# @private
|
9
|
+
class StrategyCollection
|
10
|
+
include Enumerable
|
11
|
+
|
12
|
+
attr_reader :strategies
|
13
|
+
|
14
|
+
# @param [Hash, Array, nil] strategies Concurrency or Threshold options
|
15
|
+
# or array of options.
|
16
|
+
# See keyword args of {Strategy::Concurrency#initialize} for details.
|
17
|
+
# See keyword args of {Strategy::Threshold#initialize} for details.
|
18
|
+
# @param [Class] strategy class of strategy: Concurrency or Threshold
|
19
|
+
# @param [#to_s] name
|
20
|
+
# @param [#call] key_suffix Dynamic key suffix generator.
|
21
|
+
def initialize(strategies, strategy:, name:, key_suffix:)
|
22
|
+
strategies = (strategies.is_a?(Hash) ? [strategies] : Array(strategies))
|
23
|
+
@strategies = strategies.map do |options|
|
24
|
+
make_strategy(strategy, name, key_suffix, options)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
# @param [#call] block
|
29
|
+
# Iterates each strategy in collection
|
30
|
+
def each(&block)
|
31
|
+
@strategies.each(&block)
|
32
|
+
end
|
33
|
+
|
34
|
+
# @return [Boolean] whenever any strategy in collection has dynamic config
|
35
|
+
def dynamic?
|
36
|
+
any?(&:dynamic?)
|
37
|
+
end
|
38
|
+
|
39
|
+
# @return [Boolean] whenever job is throttled or not
|
40
|
+
# by any strategy in collection.
|
41
|
+
def throttled?(*args)
|
42
|
+
any? { |s| s.throttled?(*args) }
|
43
|
+
end
|
44
|
+
|
45
|
+
# Marks job as being processed.
|
46
|
+
# @return [void]
|
47
|
+
def finalize!(*args)
|
48
|
+
each { |c| c.finalize!(*args) }
|
49
|
+
end
|
50
|
+
|
51
|
+
# Resets count of jobs of all avaliable strategies
|
52
|
+
# @return [void]
|
53
|
+
def reset!
|
54
|
+
each(&:reset!)
|
55
|
+
end
|
56
|
+
|
57
|
+
private
|
58
|
+
|
59
|
+
# @return [Base, nil]
|
60
|
+
def make_strategy(strategy, name, key_suffix, options)
|
61
|
+
return unless options
|
62
|
+
|
63
|
+
strategy.new("throttled:#{name}", {
|
64
|
+
:key_suffix => key_suffix,
|
65
|
+
**options
|
66
|
+
})
|
67
|
+
end
|
68
|
+
end
|
69
|
+
end
|
70
|
+
end
|
@@ -9,7 +9,7 @@ module Sidekiq
|
|
9
9
|
# @param name [#to_s] Constant name
|
10
10
|
# @return [Object, nil] Resolved constant or nil if failed.
|
11
11
|
def constantize(name)
|
12
|
-
name.to_s.sub(
|
12
|
+
name.to_s.sub(%r{^::}, "").split("::").inject(Object, &:const_get)
|
13
13
|
rescue NameError
|
14
14
|
Sidekiq.logger.warn { "Failed to constantize: #{name}" }
|
15
15
|
nil
|
@@ -4,7 +4,7 @@ module Sidekiq
|
|
4
4
|
module Throttled
|
5
5
|
module Web
|
6
6
|
module SummaryFix
|
7
|
-
JAVASCRIPT = [File.read(
|
7
|
+
JAVASCRIPT = [File.read(File.expand_path("summary_fix.js", __dir__)).freeze].freeze
|
8
8
|
HEADERS = { "Content-Type" => "application/javascript" }.freeze
|
9
9
|
|
10
10
|
class << self
|
@@ -14,10 +14,14 @@
|
|
14
14
|
<tr>
|
15
15
|
<td style="vertical-align:middle;"><%= name %></td>
|
16
16
|
<td style="vertical-align:middle;text-align:center;">
|
17
|
-
|
17
|
+
<% strategy.concurrency.each do |concurrency| %>
|
18
|
+
<%= Sidekiq::Throttled::Web::Stats.new(concurrency).to_html %>
|
19
|
+
<% end %>
|
18
20
|
</td>
|
19
21
|
<td style="vertical-align:middle;text-align:center;">
|
20
|
-
|
22
|
+
<% strategy.threshold.each do |threshold| %>
|
23
|
+
<%= Sidekiq::Throttled::Web::Stats.new(threshold).to_html %>
|
24
|
+
<% end %>
|
21
25
|
</td>
|
22
26
|
<td style="vertical-align:middle;text-align:center;">
|
23
27
|
<form action="<%= root_path %>throttled/<%= CGI.escape name %>/reset" method="post">
|
@@ -77,7 +77,7 @@ module Sidekiq
|
|
77
77
|
Registry.add(self, **kwargs)
|
78
78
|
end
|
79
79
|
|
80
|
-
# Adds current worker to preconfigured
|
80
|
+
# Adds current worker to preconfigured throttling strategy. Allows
|
81
81
|
# sharing same pool for multiple workers.
|
82
82
|
#
|
83
83
|
# First of all we need to create shared throttling strategy:
|
@@ -85,7 +85,7 @@ module Sidekiq
|
|
85
85
|
# # Create google_api throttling strategy
|
86
86
|
# Sidekiq::Throttled::Registry.add(:google_api, {
|
87
87
|
# :threshold => { :limit => 123, :period => 1.hour },
|
88
|
-
# :concurrency => { :limit =>
|
88
|
+
# :concurrency => { :limit => 10 }
|
89
89
|
# })
|
90
90
|
#
|
91
91
|
# Now we can assign it to our workers:
|
data/sidekiq-throttled.gemspec
CHANGED
@@ -24,6 +24,8 @@ Gem::Specification.new do |spec|
|
|
24
24
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
25
25
|
spec.require_paths = ["lib"]
|
26
26
|
|
27
|
+
spec.required_ruby_version = "~> 2.6"
|
28
|
+
|
27
29
|
spec.add_runtime_dependency "concurrent-ruby"
|
28
30
|
spec.add_runtime_dependency "redis-prescription"
|
29
31
|
spec.add_runtime_dependency "sidekiq"
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-throttled
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.14.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Alexey V Zapparov
|
8
|
-
autorequire:
|
8
|
+
autorequire:
|
9
9
|
bindir: exe
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2021-09-21 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: concurrent-ruby
|
@@ -78,6 +78,12 @@ files:
|
|
78
78
|
- ".gitignore"
|
79
79
|
- ".rspec"
|
80
80
|
- ".rubocop.yml"
|
81
|
+
- ".rubocop/layout.yml"
|
82
|
+
- ".rubocop/lint.yml"
|
83
|
+
- ".rubocop/metrics.yml"
|
84
|
+
- ".rubocop/performance.yml"
|
85
|
+
- ".rubocop/rspec.yml"
|
86
|
+
- ".rubocop/style.yml"
|
81
87
|
- ".rubocop_todo.yml"
|
82
88
|
- ".travis.yml"
|
83
89
|
- ".yardopts"
|
@@ -113,6 +119,7 @@ files:
|
|
113
119
|
- lib/sidekiq/throttled/strategy/concurrency.rb
|
114
120
|
- lib/sidekiq/throttled/strategy/threshold.lua
|
115
121
|
- lib/sidekiq/throttled/strategy/threshold.rb
|
122
|
+
- lib/sidekiq/throttled/strategy_collection.rb
|
116
123
|
- lib/sidekiq/throttled/testing.rb
|
117
124
|
- lib/sidekiq/throttled/utils.rb
|
118
125
|
- lib/sidekiq/throttled/version.rb
|
@@ -128,23 +135,23 @@ homepage: https://github.com/sensortower/sidekiq-throttled
|
|
128
135
|
licenses:
|
129
136
|
- MIT
|
130
137
|
metadata: {}
|
131
|
-
post_install_message:
|
138
|
+
post_install_message:
|
132
139
|
rdoc_options: []
|
133
140
|
require_paths:
|
134
141
|
- lib
|
135
142
|
required_ruby_version: !ruby/object:Gem::Requirement
|
136
143
|
requirements:
|
137
|
-
- - "
|
144
|
+
- - "~>"
|
138
145
|
- !ruby/object:Gem::Version
|
139
|
-
version: '
|
146
|
+
version: '2.6'
|
140
147
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
141
148
|
requirements:
|
142
149
|
- - ">="
|
143
150
|
- !ruby/object:Gem::Version
|
144
151
|
version: '0'
|
145
152
|
requirements: []
|
146
|
-
rubygems_version: 3.1.
|
147
|
-
signing_key:
|
153
|
+
rubygems_version: 3.1.4
|
154
|
+
signing_key:
|
148
155
|
specification_version: 4
|
149
156
|
summary: Concurrency and threshold throttling for Sidekiq.
|
150
157
|
test_files: []
|