sidekiq-throttled 0.6.6 → 0.6.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.todo.yml +25 -0
- data/.rubocop.yml +25 -46
- data/.travis.yml +41 -20
- data/Appraisals +2 -2
- data/CHANGES.md +6 -0
- data/Gemfile +9 -2
- data/Guardfile +25 -0
- data/LICENSE.md +1 -1
- data/README.md +31 -3
- data/Rakefile +19 -3
- data/gemfiles/sidekiq_4.0.gemfile +8 -2
- data/gemfiles/sidekiq_4.1.gemfile +8 -2
- data/gemfiles/{sidekiq_latest.gemfile → sidekiq_4.2.gemfile} +9 -3
- data/lib/sidekiq/throttled.rb +2 -2
- data/lib/sidekiq/throttled/communicator.rb +1 -1
- data/lib/sidekiq/throttled/communicator/callbacks.rb +1 -1
- data/lib/sidekiq/throttled/communicator/listener.rb +2 -2
- data/lib/sidekiq/throttled/expirable_list.rb +67 -0
- data/lib/sidekiq/throttled/fetch.rb +7 -3
- data/lib/sidekiq/throttled/middleware.rb +2 -2
- data/lib/sidekiq/throttled/queue_name.rb +2 -2
- data/lib/sidekiq/throttled/queues_pauser.rb +3 -3
- data/lib/sidekiq/throttled/strategy/concurrency.rb +1 -1
- data/lib/sidekiq/throttled/strategy/script.rb +3 -4
- data/lib/sidekiq/throttled/strategy/threshold.rb +1 -1
- data/lib/sidekiq/throttled/version.rb +1 -1
- data/lib/sidekiq/throttled/web/stats.rb +3 -4
- metadata +7 -4
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA1:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: ec534fed2e493758df139b6245cf1f9e657c2e18
|
|
4
|
+
data.tar.gz: 3de82b6c63e63e59a7c14c47683b9a9bff3dd912
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: cb31fced1ceae5050599471b39cbe8eedd7285bab72b77923fc471e4104eb1fdb85b7631622917ce423d3440abf06953123f289ce9d1917d1a33446beb39f3d2
|
|
7
|
+
data.tar.gz: eb34237d9b64bb9c3cc4b6a9d33dc79d439b7990e84a59d63a999361eed857eda4cdf100831fd6e1e1f944d118d7dda504c98980d7bc33aaeafe0c01af3bcefd
|
data/.rubocop.todo.yml
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# need to review if it's possible to slim down methods down to 10 LOCs
|
|
2
|
+
# and if not, either disable this cop inline or bump up limits. in any
|
|
3
|
+
# case once we're clear on max allowed LOCs per method - it should be
|
|
4
|
+
# fixed in rubocop config (as rubocop constantly changing "best practices"
|
|
5
|
+
# according to moon's phase and wind's direction).
|
|
6
|
+
Metrics/MethodLength:
|
|
7
|
+
Exclude:
|
|
8
|
+
- lib/sidekiq/throttled/strategy.rb
|
|
9
|
+
- lib/sidekiq/throttled/queues_pauser.rb
|
|
10
|
+
- lib/sidekiq/throttled/communicator/callbacks.rb
|
|
11
|
+
- lib/sidekiq/throttled/communicator/listener.rb
|
|
12
|
+
|
|
13
|
+
# i have no strong feeling one way or the other, but probably using JSON for
|
|
14
|
+
# serializatino will be a bit more secure indeed.
|
|
15
|
+
Security/MarshalLoad:
|
|
16
|
+
Exclude:
|
|
17
|
+
- lib/sidekiq/throttled/communicator/listener.rb
|
|
18
|
+
|
|
19
|
+
# Enable this cop once we drop Ruby 2.2.x support.
|
|
20
|
+
Style/SafeNavigation:
|
|
21
|
+
Enabled: false
|
|
22
|
+
|
|
23
|
+
# Enable this cop once we drop Ruby 2.2.x support.
|
|
24
|
+
Style/NumericPredicate:
|
|
25
|
+
Enabled: false
|
data/.rubocop.yml
CHANGED
|
@@ -1,32 +1,35 @@
|
|
|
1
|
+
inherit_from: .rubocop.todo.yml
|
|
2
|
+
|
|
3
|
+
################################################################################
|
|
4
|
+
|
|
1
5
|
AllCops:
|
|
2
6
|
DisplayCopNames: true
|
|
7
|
+
TargetRubyVersion: 2.4
|
|
8
|
+
|
|
9
|
+
## Metrics #####################################################################
|
|
10
|
+
|
|
11
|
+
Metrics/BlockLength:
|
|
12
|
+
Exclude:
|
|
13
|
+
- "Guardfile"
|
|
14
|
+
- "spec/**/*"
|
|
3
15
|
|
|
4
16
|
## Styles ######################################################################
|
|
5
17
|
|
|
18
|
+
Style/AlignHash:
|
|
19
|
+
EnforcedHashRocketStyle: table
|
|
20
|
+
|
|
6
21
|
Style/AlignParameters:
|
|
7
22
|
EnforcedStyle: with_fixed_indentation
|
|
8
23
|
|
|
9
24
|
Style/BracesAroundHashParameters:
|
|
10
25
|
Enabled: false
|
|
11
26
|
|
|
12
|
-
# Broken (2014-12-15). Use `yardstick` gem instead.
|
|
13
|
-
# See: https://github.com/bbatsov/rubocop/issues/947
|
|
14
|
-
# TODO: Enable back once cop is fixed.
|
|
15
27
|
Style/Documentation:
|
|
16
28
|
Enabled: false
|
|
17
29
|
|
|
18
|
-
Style/EmptyCaseCondition:
|
|
19
|
-
Enabled: false
|
|
20
|
-
|
|
21
|
-
Style/EmptyLineBetweenDefs:
|
|
22
|
-
AllowAdjacentOneLineDefs: true
|
|
23
|
-
|
|
24
30
|
Style/Encoding:
|
|
25
31
|
EnforcedStyle: when_needed
|
|
26
32
|
|
|
27
|
-
Style/FrozenStringLiteralComment:
|
|
28
|
-
EnforcedStyle: always
|
|
29
|
-
|
|
30
33
|
Style/HashSyntax:
|
|
31
34
|
EnforcedStyle: hash_rockets
|
|
32
35
|
|
|
@@ -36,48 +39,24 @@ Style/IndentArray:
|
|
|
36
39
|
Style/IndentHash:
|
|
37
40
|
EnforcedStyle: consistent
|
|
38
41
|
|
|
39
|
-
#
|
|
42
|
+
# Follow your heart where it makes sense to use lambda or lambda literal.
|
|
43
|
+
# Enforcing it makes some pieces of code look REALLY terrible, e.g. in
|
|
44
|
+
# case of empty (noop) lambdas: `lambda { |_| }`.
|
|
40
45
|
Style/Lambda:
|
|
41
46
|
Enabled: false
|
|
42
47
|
|
|
43
|
-
Style/
|
|
48
|
+
Style/MultilineMethodCallIndentation:
|
|
44
49
|
EnforcedStyle: indented
|
|
45
50
|
|
|
46
|
-
#
|
|
47
|
-
#
|
|
48
|
-
#
|
|
49
|
-
#
|
|
51
|
+
# Enabling this cop makes Guardfile (which is full of pathname regexps)
|
|
52
|
+
# look absolutley style-inconsistent and terrible. In any case, this should
|
|
53
|
+
# be on developer's choice whenever to use `%r` or not. Just like we don't
|
|
54
|
+
# enforce to use `["foo"]` over `%w(foo)` and so on.
|
|
50
55
|
Style/RegexpLiteral:
|
|
51
56
|
Enabled: false
|
|
52
57
|
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
# redis do |conn|
|
|
56
|
-
# conn.hset :k1, now
|
|
57
|
-
# conn.hincrby :k2, 123
|
|
58
|
-
# end
|
|
59
|
-
Style/SpaceBeforeFirstArg:
|
|
60
|
-
Enabled: false
|
|
58
|
+
Style/SpaceInLambdaLiteral:
|
|
59
|
+
EnforcedStyle: require_space
|
|
61
60
|
|
|
62
61
|
Style/StringLiterals:
|
|
63
62
|
EnforcedStyle: double_quotes
|
|
64
|
-
|
|
65
|
-
# Not all trivial readers/writers can be defined with attr_* methods
|
|
66
|
-
#
|
|
67
|
-
# class Example < SimpleDelegator
|
|
68
|
-
# def __getobj__
|
|
69
|
-
# @obj
|
|
70
|
-
# end
|
|
71
|
-
#
|
|
72
|
-
# def __setobj__(obj)
|
|
73
|
-
# @obj = obj
|
|
74
|
-
# end
|
|
75
|
-
# end
|
|
76
|
-
Style/TrivialAccessors:
|
|
77
|
-
Enabled: false
|
|
78
|
-
|
|
79
|
-
## Metrics #####################################################################
|
|
80
|
-
|
|
81
|
-
Metrics/MethodLength:
|
|
82
|
-
CountComments: false
|
|
83
|
-
Max: 15
|
data/.travis.yml
CHANGED
|
@@ -1,27 +1,48 @@
|
|
|
1
1
|
language: ruby
|
|
2
|
+
sudo: false
|
|
3
|
+
|
|
4
|
+
services:
|
|
5
|
+
- redis-server
|
|
6
|
+
|
|
7
|
+
cache: bundler
|
|
8
|
+
|
|
9
|
+
before_install:
|
|
10
|
+
- gem update --system
|
|
11
|
+
- gem --version
|
|
12
|
+
- gem install bundler --no-rdoc --no-ri
|
|
13
|
+
- bundle --version
|
|
14
|
+
|
|
15
|
+
install: bundle install --without development
|
|
16
|
+
|
|
17
|
+
env:
|
|
18
|
+
- WITH_REDIS_NAMESPACE="false"
|
|
19
|
+
- WITH_REDIS_NAMESPACE="true"
|
|
20
|
+
|
|
2
21
|
rvm:
|
|
3
|
-
- 2.2.
|
|
4
|
-
- 2.3.
|
|
5
|
-
-
|
|
6
|
-
|
|
7
|
-
- jruby-head
|
|
8
|
-
- rbx-2
|
|
22
|
+
- 2.2.6
|
|
23
|
+
- 2.3.3
|
|
24
|
+
- 2.4.0
|
|
25
|
+
|
|
9
26
|
matrix:
|
|
10
|
-
allow_failures:
|
|
11
|
-
- rvm: ruby-head
|
|
12
|
-
- rvm: jruby-9.0.5.0
|
|
13
|
-
- rvm: jruby-head
|
|
14
|
-
- rvm: rbx-2
|
|
15
27
|
fast_finish: true
|
|
28
|
+
include:
|
|
29
|
+
-
|
|
30
|
+
rvm: 2.4.0
|
|
31
|
+
env: SUITE="rubocop"
|
|
32
|
+
gemfile: Gemfile
|
|
33
|
+
-
|
|
34
|
+
rvm: jruby-9.1.8.0
|
|
35
|
+
env: JRUBY_OPTS="$JRUBY_OPTS --debug" WITH_REDIS_NAMESPACE="false"
|
|
36
|
+
gemfile: Gemfile
|
|
37
|
+
-
|
|
38
|
+
rvm: jruby-9.1.8.0
|
|
39
|
+
env: JRUBY_OPTS="$JRUBY_OPTS --debug" WITH_REDIS_NAMESPACE="true"
|
|
40
|
+
gemfile: Gemfile
|
|
41
|
+
allow_failures:
|
|
42
|
+
-
|
|
43
|
+
rvm: jruby-9.1.8.0
|
|
44
|
+
|
|
16
45
|
gemfile:
|
|
17
46
|
- gemfiles/sidekiq_4.0.gemfile
|
|
18
47
|
- gemfiles/sidekiq_4.1.gemfile
|
|
19
|
-
- gemfiles/
|
|
20
|
-
env:
|
|
21
|
-
- WITH_REDIS_NAMESPACE=0
|
|
22
|
-
- WITH_REDIS_NAMESPACE=1
|
|
23
|
-
before_install:
|
|
24
|
-
- gem install bundler -v 1.10.6
|
|
25
|
-
services:
|
|
26
|
-
- redis-server
|
|
27
|
-
sudo: false
|
|
48
|
+
- gemfiles/sidekiq_4.2.gemfile
|
data/Appraisals
CHANGED
data/CHANGES.md
CHANGED
data/Gemfile
CHANGED
|
@@ -4,9 +4,16 @@ source "https://rubygems.org"
|
|
|
4
4
|
|
|
5
5
|
gem "appraisal"
|
|
6
6
|
gem "rake"
|
|
7
|
-
gem "rspec"
|
|
8
|
-
gem "rubocop", "~> 0.42.0", :require => false
|
|
9
7
|
gem "redis-namespace", :require => false
|
|
8
|
+
gem "rspec"
|
|
9
|
+
gem "rubocop", "~> 0.47.0", :require => false
|
|
10
|
+
gem "sidekiq"
|
|
11
|
+
|
|
12
|
+
group :development do
|
|
13
|
+
gem "guard", :require => false
|
|
14
|
+
gem "guard-rspec", :require => false
|
|
15
|
+
gem "guard-rubocop", :require => false
|
|
16
|
+
end
|
|
10
17
|
|
|
11
18
|
group :test do
|
|
12
19
|
gem "coveralls", :require => false
|
data/Guardfile
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
group :red_green_refactor, :halt_on_fail => true do
|
|
4
|
+
guard :rspec, :cmd => "bundle exec rspec --no-profile" do
|
|
5
|
+
require "guard/rspec/dsl"
|
|
6
|
+
dsl = Guard::RSpec::Dsl.new(self)
|
|
7
|
+
|
|
8
|
+
# Feel free to open issues for suggestions and improvements
|
|
9
|
+
|
|
10
|
+
# RSpec files
|
|
11
|
+
rspec = dsl.rspec
|
|
12
|
+
watch(rspec.spec_helper) { rspec.spec_dir }
|
|
13
|
+
watch(rspec.spec_support) { rspec.spec_dir }
|
|
14
|
+
watch(rspec.spec_files)
|
|
15
|
+
|
|
16
|
+
# Ruby files
|
|
17
|
+
ruby = dsl.ruby
|
|
18
|
+
dsl.watch_spec_files_for(ruby.lib_files)
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
guard :rubocop, :all_on_start => false do
|
|
22
|
+
watch(%r{.+\.rb$})
|
|
23
|
+
watch(%r{(?:.+/)?\.rubocop\.yml$}) { |m| File.dirname(m[0]) }
|
|
24
|
+
end
|
|
25
|
+
end
|
data/LICENSE.md
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
The MIT License (MIT)
|
|
2
2
|
|
|
3
|
-
Copyright (c) 2015-
|
|
3
|
+
Copyright (c) 2015-2017 SensorTower Inc.
|
|
4
4
|
|
|
5
5
|
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
6
|
of this software and associated documentation files (the "Software"), to deal
|
data/README.md
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
[](http://rubygems.org/gems/sidekiq-throttled)
|
|
4
4
|
[](https://travis-ci.org/sensortower/sidekiq-throttled)
|
|
5
5
|
[](https://codeclimate.com/github/sensortower/sidekiq-throttled)
|
|
6
|
-
[](https://coveralls.io/github/sensortower/sidekiq-throttled?branch=master)
|
|
7
7
|
[](http://inch-ci.org/github/sensortower/sidekiq-throttled)
|
|
8
8
|
|
|
9
9
|
Concurrency and threshold throttling for [Sidekiq][sidekiq].
|
|
@@ -122,8 +122,22 @@ some trouble.
|
|
|
122
122
|
This library aims to support and is [tested against][travis] the following Ruby
|
|
123
123
|
versions:
|
|
124
124
|
|
|
125
|
-
* Ruby 2.2.
|
|
125
|
+
* Ruby 2.2.6+
|
|
126
126
|
* Ruby 2.3.x
|
|
127
|
+
* Ruby 2.4.x
|
|
128
|
+
|
|
129
|
+
If something doesn't work on one of these versions, it's a bug.
|
|
130
|
+
|
|
131
|
+
This library may inadvertently work (or seem to work) on other Ruby versions,
|
|
132
|
+
however support will only be provided for the versions listed above.
|
|
133
|
+
|
|
134
|
+
If you would like this library to support another Ruby version or
|
|
135
|
+
implementation, you may volunteer to be a maintainer. Being a maintainer
|
|
136
|
+
entails making sure all tests run and pass on that implementation. When
|
|
137
|
+
something breaks on your implementation, you will be responsible for providing
|
|
138
|
+
patches in a timely fashion. If critical issues for a particular implementation
|
|
139
|
+
exist at the time of a major release, support for that Ruby version may be
|
|
140
|
+
dropped.
|
|
127
141
|
|
|
128
142
|
|
|
129
143
|
## Supported Sidekiq Versions
|
|
@@ -132,6 +146,7 @@ This library aims to support work with following [Sidekiq][sidekiq] versions:
|
|
|
132
146
|
|
|
133
147
|
* Sidekiq 4.0.x
|
|
134
148
|
* Sidekiq 4.1.x
|
|
149
|
+
* Sidekiq 4.2.x
|
|
135
150
|
|
|
136
151
|
|
|
137
152
|
## Contributing
|
|
@@ -144,9 +159,22 @@ This library aims to support work with following [Sidekiq][sidekiq] versions:
|
|
|
144
159
|
* If we've accepted a patch, feel free to ask for commit access!
|
|
145
160
|
|
|
146
161
|
|
|
162
|
+
## Development
|
|
163
|
+
|
|
164
|
+
```
|
|
165
|
+
bundle update
|
|
166
|
+
appraisal install # install dependencies for all gemfiles
|
|
167
|
+
appraisal update # update dependencies for all gemfiles
|
|
168
|
+
appraisal rspec # run rspec against each gemfile
|
|
169
|
+
bundle exec rubocop # run static code analysis
|
|
170
|
+
```
|
|
171
|
+
|
|
172
|
+
Don't forget to run `appraisal update` after any changes to `Gemfile`.
|
|
173
|
+
|
|
174
|
+
|
|
147
175
|
## Copyright
|
|
148
176
|
|
|
149
|
-
Copyright (c) 2015-
|
|
177
|
+
Copyright (c) 2015-2017 SensorTower Inc.
|
|
150
178
|
See LICENSE.md for further details.
|
|
151
179
|
|
|
152
180
|
|
data/Rakefile
CHANGED
|
@@ -4,7 +4,23 @@ require "bundler/gem_tasks"
|
|
|
4
4
|
require "rspec/core/rake_task"
|
|
5
5
|
RSpec::Core::RakeTask.new(:spec)
|
|
6
6
|
|
|
7
|
-
|
|
8
|
-
|
|
7
|
+
desc "Run RuboCop"
|
|
8
|
+
task :rubocop do
|
|
9
|
+
require "rubocop"
|
|
10
|
+
result = RuboCop::CLI.new.run([])
|
|
11
|
+
abort("RuboCop failed!") if result.nonzero?
|
|
12
|
+
end
|
|
9
13
|
|
|
10
|
-
|
|
14
|
+
namespace :rubocop do
|
|
15
|
+
desc "Auto-correct RuboCop offenses"
|
|
16
|
+
task :autocorrect do
|
|
17
|
+
require "rubocop"
|
|
18
|
+
result = RuboCop::CLI.new.run(["--auto-correct"])
|
|
19
|
+
abort("RuboCop failed!") if result.nonzero?
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
default_suite = ENV["CI"] ? :spec : %i(spec rubocop)
|
|
24
|
+
named_suites = { "rubocop" => :rubocop, "rspec" => :spec }
|
|
25
|
+
|
|
26
|
+
task :default => named_suites.fetch(ENV["SUITE"], default_suite)
|
|
@@ -4,11 +4,17 @@ source "https://rubygems.org"
|
|
|
4
4
|
|
|
5
5
|
gem "appraisal"
|
|
6
6
|
gem "rake"
|
|
7
|
-
gem "rspec"
|
|
8
|
-
gem "rubocop", "~> 0.42.0", :require => false
|
|
9
7
|
gem "redis-namespace", :require => false
|
|
8
|
+
gem "rspec"
|
|
9
|
+
gem "rubocop", "~> 0.47.0", :require => false
|
|
10
10
|
gem "sidekiq", "~> 4.0.0"
|
|
11
11
|
|
|
12
|
+
group :development do
|
|
13
|
+
gem "guard", :require => false
|
|
14
|
+
gem "guard-rspec", :require => false
|
|
15
|
+
gem "guard-rubocop", :require => false
|
|
16
|
+
end
|
|
17
|
+
|
|
12
18
|
group :test do
|
|
13
19
|
gem "coveralls", :require => false
|
|
14
20
|
gem "rack-test"
|
|
@@ -4,11 +4,17 @@ source "https://rubygems.org"
|
|
|
4
4
|
|
|
5
5
|
gem "appraisal"
|
|
6
6
|
gem "rake"
|
|
7
|
-
gem "rspec"
|
|
8
|
-
gem "rubocop", "~> 0.42.0", :require => false
|
|
9
7
|
gem "redis-namespace", :require => false
|
|
8
|
+
gem "rspec"
|
|
9
|
+
gem "rubocop", "~> 0.47.0", :require => false
|
|
10
10
|
gem "sidekiq", "~> 4.1.0"
|
|
11
11
|
|
|
12
|
+
group :development do
|
|
13
|
+
gem "guard", :require => false
|
|
14
|
+
gem "guard-rspec", :require => false
|
|
15
|
+
gem "guard-rubocop", :require => false
|
|
16
|
+
end
|
|
17
|
+
|
|
12
18
|
group :test do
|
|
13
19
|
gem "coveralls", :require => false
|
|
14
20
|
gem "rack-test"
|
|
@@ -4,10 +4,16 @@ source "https://rubygems.org"
|
|
|
4
4
|
|
|
5
5
|
gem "appraisal"
|
|
6
6
|
gem "rake"
|
|
7
|
-
gem "rspec"
|
|
8
|
-
gem "rubocop", "~> 0.42.0", :require => false
|
|
9
7
|
gem "redis-namespace", :require => false
|
|
10
|
-
gem "
|
|
8
|
+
gem "rspec"
|
|
9
|
+
gem "rubocop", "~> 0.47.0", :require => false
|
|
10
|
+
gem "sidekiq", "~> 4.2.0"
|
|
11
|
+
|
|
12
|
+
group :development do
|
|
13
|
+
gem "guard", :require => false
|
|
14
|
+
gem "guard-rspec", :require => false
|
|
15
|
+
gem "guard-rubocop", :require => false
|
|
16
|
+
end
|
|
11
17
|
|
|
12
18
|
group :test do
|
|
13
19
|
gem "coveralls", :require => false
|
data/lib/sidekiq/throttled.rb
CHANGED
|
@@ -79,8 +79,8 @@ module Sidekiq
|
|
|
79
79
|
# @return [Boolean]
|
|
80
80
|
def throttled?(message)
|
|
81
81
|
message = JSON.parse message
|
|
82
|
-
job = message.fetch("class"
|
|
83
|
-
jid = message.fetch("jid"
|
|
82
|
+
job = message.fetch("class") { return false }
|
|
83
|
+
jid = message.fetch("jid") { return false }
|
|
84
84
|
|
|
85
85
|
Registry.get job do |strategy|
|
|
86
86
|
return strategy.throttled?(jid, *message["args"])
|
|
@@ -69,12 +69,12 @@ module Sidekiq
|
|
|
69
69
|
@subscribed = false
|
|
70
70
|
rescue StandardError => e
|
|
71
71
|
@subscribed = false
|
|
72
|
-
handle_exception(e, { :context => "sidekiq:throttled"
|
|
72
|
+
handle_exception(e, { :context => "sidekiq:throttled" })
|
|
73
73
|
sleep 1
|
|
74
74
|
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
75
75
|
@terminated = true
|
|
76
76
|
@subscribed = false
|
|
77
|
-
handle_exception(e, { :context => "sidekiq:throttled"
|
|
77
|
+
handle_exception(e, { :context => "sidekiq:throttled" })
|
|
78
78
|
raise
|
|
79
79
|
end
|
|
80
80
|
|
|
@@ -0,0 +1,67 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "monitor"
|
|
4
|
+
|
|
5
|
+
module Sidekiq
|
|
6
|
+
module Throttled
|
|
7
|
+
# List that tracks when elements were added and enumerates over those not
|
|
8
|
+
# older than `ttl` seconds ago.
|
|
9
|
+
#
|
|
10
|
+
# ## Implementation
|
|
11
|
+
#
|
|
12
|
+
# Internally list holds an array of arrays. Thus ecah element is a tuple of
|
|
13
|
+
# timestamp (when element was added) and element itself:
|
|
14
|
+
#
|
|
15
|
+
# [
|
|
16
|
+
# [ 1234567890.12345, "default" ],
|
|
17
|
+
# [ 1234567890.34567, "urgent" ],
|
|
18
|
+
# [ 1234579621.56789, "urgent" ],
|
|
19
|
+
# ...
|
|
20
|
+
# ]
|
|
21
|
+
#
|
|
22
|
+
# It does not deduplicates elements. Eviction happens only upon elements
|
|
23
|
+
# retrieval (see {#each}).
|
|
24
|
+
#
|
|
25
|
+
# @private
|
|
26
|
+
class ExpirableList
|
|
27
|
+
include Enumerable
|
|
28
|
+
|
|
29
|
+
# @param ttl [Float] elements time-to-live in seconds
|
|
30
|
+
def initialize(ttl)
|
|
31
|
+
@ttl = ttl.to_f
|
|
32
|
+
@arr = []
|
|
33
|
+
@mon = Monitor.new
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Pushes given element into the list.
|
|
37
|
+
#
|
|
38
|
+
# @params element [Object]
|
|
39
|
+
# @return [ExpirableList] self
|
|
40
|
+
def <<(element)
|
|
41
|
+
@mon.synchronize { @arr << [Time.now.to_f, element] }
|
|
42
|
+
self
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
# Evicts expired elements and calls the given block once for each element
|
|
46
|
+
# left, passing that element as a parameter.
|
|
47
|
+
#
|
|
48
|
+
# @yield [element]
|
|
49
|
+
# @return [Enumerator] if no block given
|
|
50
|
+
# @return [ExpirableList] self if block given
|
|
51
|
+
def each
|
|
52
|
+
return to_enum __method__ unless block_given?
|
|
53
|
+
|
|
54
|
+
@mon.synchronize do
|
|
55
|
+
horizon = Time.now.to_f - @ttl
|
|
56
|
+
|
|
57
|
+
# drop all elements older than horizon
|
|
58
|
+
@arr.shift while @arr[0] && @arr[0][0] < horizon
|
|
59
|
+
|
|
60
|
+
@arr.each { |x| yield x[1] }
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
self
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
end
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "sidekiq"
|
|
4
|
+
require "sidekiq/throttled/expirable_list"
|
|
4
5
|
require "sidekiq/throttled/fetch/unit_of_work"
|
|
5
6
|
require "sidekiq/throttled/queues_pauser"
|
|
6
7
|
require "sidekiq/throttled/queue_name"
|
|
@@ -11,11 +12,14 @@ module Sidekiq
|
|
|
11
12
|
#
|
|
12
13
|
# @private
|
|
13
14
|
class Fetch
|
|
15
|
+
# Timeout to sleep between fetch retries in case of no job received,
|
|
16
|
+
# as well as timeout to wait for redis to give us something to work.
|
|
14
17
|
TIMEOUT = 2
|
|
15
|
-
private_constant :TIMEOUT
|
|
16
18
|
|
|
17
19
|
# Initializes fetcher instance.
|
|
18
20
|
def initialize(options)
|
|
21
|
+
@paused = ExpirableList.new(TIMEOUT)
|
|
22
|
+
|
|
19
23
|
@strict = options[:strict]
|
|
20
24
|
@queues = options[:queues].map { |q| QueueName.expand q }
|
|
21
25
|
|
|
@@ -33,7 +37,7 @@ module Sidekiq
|
|
|
33
37
|
return work unless work.throttled?
|
|
34
38
|
|
|
35
39
|
work.requeue_throttled
|
|
36
|
-
|
|
40
|
+
@paused << QueueName.expand(work.queue_name)
|
|
37
41
|
|
|
38
42
|
nil
|
|
39
43
|
end
|
|
@@ -78,7 +82,7 @@ module Sidekiq
|
|
|
78
82
|
# @param [Array<String>] queues
|
|
79
83
|
# @return [Array<String>]
|
|
80
84
|
def filter_queues(queues)
|
|
81
|
-
QueuesPauser.instance.filter(queues)
|
|
85
|
+
QueuesPauser.instance.filter(queues) - @paused.to_a
|
|
82
86
|
end
|
|
83
87
|
end
|
|
84
88
|
end
|
|
@@ -12,8 +12,8 @@ module Sidekiq
|
|
|
12
12
|
def call(_worker, msg, _queue)
|
|
13
13
|
yield
|
|
14
14
|
ensure
|
|
15
|
-
Registry.get msg["class"
|
|
16
|
-
strategy.finalize!(msg["jid"
|
|
15
|
+
Registry.get msg["class"] do |strategy|
|
|
16
|
+
strategy.finalize!(msg["jid"], *msg["args"])
|
|
17
17
|
end
|
|
18
18
|
end
|
|
19
19
|
end
|
|
@@ -27,7 +27,7 @@ module Sidekiq
|
|
|
27
27
|
# @param [String]
|
|
28
28
|
# @return [String]
|
|
29
29
|
def normalize(queue)
|
|
30
|
-
queue.sub(QUEUE_NAME_PREFIX_RE, ""
|
|
30
|
+
queue.sub(QUEUE_NAME_PREFIX_RE, "")
|
|
31
31
|
end
|
|
32
32
|
|
|
33
33
|
# Prepends `queue:` prefix to given `queue` name.
|
|
@@ -38,7 +38,7 @@ module Sidekiq
|
|
|
38
38
|
# @param [String] queue Queue name
|
|
39
39
|
# @return [String]
|
|
40
40
|
def expand(queue)
|
|
41
|
-
"queue:#{queue}"
|
|
41
|
+
"queue:#{queue}"
|
|
42
42
|
end
|
|
43
43
|
end
|
|
44
44
|
end
|
|
@@ -19,19 +19,19 @@ module Sidekiq
|
|
|
19
19
|
# Redis key of Set with paused queues.
|
|
20
20
|
#
|
|
21
21
|
# @return [String]
|
|
22
|
-
PAUSED_QUEUES = "throttled:X:paused_queues"
|
|
22
|
+
PAUSED_QUEUES = "throttled:X:paused_queues"
|
|
23
23
|
private_constant :PAUSED_QUEUES
|
|
24
24
|
|
|
25
25
|
# {Communicator} message used to notify that queue needs to be paused.
|
|
26
26
|
#
|
|
27
27
|
# @return [String]
|
|
28
|
-
PAUSE_MESSAGE = "pause"
|
|
28
|
+
PAUSE_MESSAGE = "pause"
|
|
29
29
|
private_constant :PAUSE_MESSAGE
|
|
30
30
|
|
|
31
31
|
# {Communicator} message used to notify that queue needs to be resumed.
|
|
32
32
|
#
|
|
33
33
|
# @return [String]
|
|
34
|
-
RESUME_MESSAGE = "resume"
|
|
34
|
+
RESUME_MESSAGE = "resume"
|
|
35
35
|
private_constant :RESUME_MESSAGE
|
|
36
36
|
|
|
37
37
|
# Initializes singleton instance.
|
|
@@ -23,7 +23,7 @@ module Sidekiq
|
|
|
23
23
|
# @param [#to_i] ttl Concurrency lock TTL in seconds.
|
|
24
24
|
# @param [Proc] key_suffix Dynamic key suffix generator.
|
|
25
25
|
def initialize(strategy_key, limit:, ttl: 900, key_suffix: nil)
|
|
26
|
-
@base_key = "#{strategy_key}:concurrency"
|
|
26
|
+
@base_key = "#{strategy_key}:concurrency"
|
|
27
27
|
@limit = limit
|
|
28
28
|
@ttl = ttl.to_i
|
|
29
29
|
@key_suffix = key_suffix
|
|
@@ -15,11 +15,11 @@ module Sidekiq
|
|
|
15
15
|
# @private
|
|
16
16
|
class Script
|
|
17
17
|
# Script load command
|
|
18
|
-
LOAD = "load"
|
|
18
|
+
LOAD = "load"
|
|
19
19
|
private_constant :LOAD
|
|
20
20
|
|
|
21
21
|
# Redis error fired when script ID is unkown
|
|
22
|
-
NOSCRIPT = "NOSCRIPT"
|
|
22
|
+
NOSCRIPT = "NOSCRIPT"
|
|
23
23
|
private_constant :NOSCRIPT
|
|
24
24
|
|
|
25
25
|
# LUA script source.
|
|
@@ -48,8 +48,7 @@ module Sidekiq
|
|
|
48
48
|
# changed in redis, which is not likely gonna happen.
|
|
49
49
|
unless @digest == digest
|
|
50
50
|
if @logger
|
|
51
|
-
@logger.warn \
|
|
52
|
-
"Unexpected script SHA1 digest: " \
|
|
51
|
+
@logger.warn "Unexpected script SHA1 digest: " \
|
|
53
52
|
"#{digest.inspect} (expected: #{@digest.inspect})"
|
|
54
53
|
end
|
|
55
54
|
|
|
@@ -35,7 +35,7 @@ module Sidekiq
|
|
|
35
35
|
# @param [#to_f, #call] :period Period in seconds.
|
|
36
36
|
# @param [Proc] key_suffix Dynamic key suffix generator.
|
|
37
37
|
def initialize(strategy_key, limit:, period:, key_suffix: nil)
|
|
38
|
-
@base_key = "#{strategy_key}:threshold"
|
|
38
|
+
@base_key = "#{strategy_key}:threshold"
|
|
39
39
|
@limit = limit
|
|
40
40
|
@period = period
|
|
41
41
|
@key_suffix = key_suffix
|
|
@@ -37,10 +37,9 @@ module Sidekiq
|
|
|
37
37
|
# @return [String]
|
|
38
38
|
def colorize_count(int, max)
|
|
39
39
|
percentile = 100.00 * int / max
|
|
40
|
-
lvl =
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
else "success"
|
|
40
|
+
lvl = if 80 <= percentile then "danger"
|
|
41
|
+
elsif 60 <= percentile then "warning"
|
|
42
|
+
else "success"
|
|
44
43
|
end
|
|
45
44
|
|
|
46
45
|
%(<span class="label label-#{lvl}">#{int}</span>)
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: sidekiq-throttled
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.6.
|
|
4
|
+
version: 0.6.7
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Alexey V Zapparov
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2017-03-20 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: sidekiq
|
|
@@ -48,23 +48,26 @@ files:
|
|
|
48
48
|
- ".coveralls.yml"
|
|
49
49
|
- ".gitignore"
|
|
50
50
|
- ".rspec"
|
|
51
|
+
- ".rubocop.todo.yml"
|
|
51
52
|
- ".rubocop.yml"
|
|
52
53
|
- ".travis.yml"
|
|
53
54
|
- ".yardopts"
|
|
54
55
|
- Appraisals
|
|
55
56
|
- CHANGES.md
|
|
56
57
|
- Gemfile
|
|
58
|
+
- Guardfile
|
|
57
59
|
- LICENSE.md
|
|
58
60
|
- README.md
|
|
59
61
|
- Rakefile
|
|
60
62
|
- gemfiles/sidekiq_4.0.gemfile
|
|
61
63
|
- gemfiles/sidekiq_4.1.gemfile
|
|
62
|
-
- gemfiles/
|
|
64
|
+
- gemfiles/sidekiq_4.2.gemfile
|
|
63
65
|
- lib/sidekiq/throttled.rb
|
|
64
66
|
- lib/sidekiq/throttled/communicator.rb
|
|
65
67
|
- lib/sidekiq/throttled/communicator/callbacks.rb
|
|
66
68
|
- lib/sidekiq/throttled/communicator/listener.rb
|
|
67
69
|
- lib/sidekiq/throttled/errors.rb
|
|
70
|
+
- lib/sidekiq/throttled/expirable_list.rb
|
|
68
71
|
- lib/sidekiq/throttled/fetch.rb
|
|
69
72
|
- lib/sidekiq/throttled/fetch/unit_of_work.rb
|
|
70
73
|
- lib/sidekiq/throttled/middleware.rb
|
|
@@ -104,7 +107,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
104
107
|
version: '0'
|
|
105
108
|
requirements: []
|
|
106
109
|
rubyforge_project:
|
|
107
|
-
rubygems_version: 2.5.
|
|
110
|
+
rubygems_version: 2.4.5.2
|
|
108
111
|
signing_key:
|
|
109
112
|
specification_version: 4
|
|
110
113
|
summary: Concurrency and threshold throttling for Sidekiq.
|