sidekiq-throttled 0.10.0.alpha → 0.13.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: 83591901b017e382b25575720fab1bbcc45ac9e81acec417da1a10cbfa333613
4
- data.tar.gz: 1a135a2b18d51b644f7713cdd1d2bcad6ffe34b87ca93b11aa1336e2e1b91a8b
3
+ metadata.gz: 2b1f0237b14be26ff2b25aeaedd80e4029610cfa7c2be331827c5b865c338f48
4
+ data.tar.gz: 3a5be191a1f02b2ce21b766be820201b6f4d06b74323456412096a0beef38d11
5
5
  SHA512:
6
- metadata.gz: 594b29b269f4b8c2351adf7f835cad04d974794ef596fdcd5492202c4e4f6af5d1a06b7ec67baf15a6d393aa2a60093cc7a536f87c8260f3681b25a817d94990
7
- data.tar.gz: d4e9d76b8627c86ee7e399e563d7c9ce8bdbabc40444349a88ce88ea3bec136974505feb9b49d905bf47fc9aaf233270e9637ebca027d533e589ca93cc6cb2f5
6
+ metadata.gz: e9da0e99d328b6825ab3f47e1ea94c255717a01d05e28c90399aaf826a8d43a0e10de82baee329ddd8d846ac4e4ff0cb6f09ef11277fef2143972c50b2c3a55b
7
+ data.tar.gz: d73d011be01f319460fe87fda608aea9550e1f560bf778dfd5628874f0f074fc2020ad5e2f36ec8413a349c6dcd620f7aaa84a48f15e59dab8bf508e3e769354
@@ -0,0 +1,56 @@
1
+ name: CI
2
+
3
+ on:
4
+ push:
5
+ branches: [ master ]
6
+ pull_request:
7
+ branches: [ master ]
8
+
9
+ jobs:
10
+ rspec:
11
+ name: "rspec (ruby:${{ matrix.ruby }} sidekiq:${{ matrix.sidekiq }})"
12
+
13
+ strategy:
14
+ matrix:
15
+ ruby: [ "2.4", "2.5", "2.6", "2.7" ]
16
+ sidekiq: [ "5.0", "5.1", "5.2", "6.0" ]
17
+ exclude:
18
+ - { ruby: "2.4", sidekiq: "6.0" }
19
+
20
+ runs-on: ubuntu-latest
21
+
22
+ services:
23
+ redis:
24
+ image: redis
25
+ ports: ["6379:6379"]
26
+ options: "--entrypoint redis-server"
27
+
28
+ env:
29
+ GEMFILE: gemfiles/sidekiq_${{ matrix.sidekiq }}.gemfile
30
+
31
+ steps:
32
+ - uses: actions/checkout@v2
33
+
34
+ - uses: ruby/setup-ruby@v1
35
+ with:
36
+ ruby-version: ${{ matrix.ruby }}
37
+
38
+ - name: bundle install
39
+ run: bundle install --without development --jobs 4 --retry 3
40
+
41
+ - run: bundle exec rspec
42
+
43
+ rubocop:
44
+ runs-on: ubuntu-latest
45
+
46
+ steps:
47
+ - uses: actions/checkout@v2
48
+
49
+ - uses: ruby/setup-ruby@v1
50
+ with:
51
+ ruby-version: "2.4"
52
+
53
+ - name: bundle install
54
+ run: bundle install --without development --jobs 4 --retry 3
55
+
56
+ - run: bundle exec rubocop
@@ -8,30 +8,47 @@ require:
8
8
 
9
9
  AllCops:
10
10
  DisplayCopNames: true
11
- TargetRubyVersion: 2.3
11
+ TargetRubyVersion: 2.4
12
12
  Exclude:
13
13
  - "gemfiles/*"
14
14
 
15
15
  ## Layout ######################################################################
16
16
 
17
- Layout/AlignHash:
18
- EnforcedHashRocketStyle: table
19
-
20
- Layout/AlignParameters:
17
+ Layout/ArgumentAlignment:
21
18
  EnforcedStyle: with_fixed_indentation
22
19
 
23
- Layout/IndentArray:
20
+ Layout/FirstArrayElementIndentation:
24
21
  EnforcedStyle: consistent
25
22
 
26
- Layout/IndentHash:
23
+ Layout/FirstHashElementIndentation:
27
24
  EnforcedStyle: consistent
28
25
 
26
+ Layout/HashAlignment:
27
+ EnforcedHashRocketStyle: table
28
+
29
+ Layout/LineLength:
30
+ Max: 100
31
+
29
32
  Layout/MultilineMethodCallIndentation:
30
33
  EnforcedStyle: indented
31
34
 
35
+ Layout/ParameterAlignment:
36
+ EnforcedStyle: with_fixed_indentation
37
+
38
+ Layout/SpaceAroundMethodCallOperator:
39
+ Enabled: true
40
+
32
41
  Layout/SpaceInLambdaLiteral:
33
42
  EnforcedStyle: require_space
34
43
 
44
+ ## Lint ########################################################################
45
+
46
+ Lint/RaiseException:
47
+ Enabled: true
48
+
49
+ Lint/StructNewOverride:
50
+ Enabled: true
51
+
35
52
  ## Metrics #####################################################################
36
53
 
37
54
  Metrics/BlockLength:
@@ -41,15 +58,24 @@ Metrics/BlockLength:
41
58
 
42
59
  ## Styles ######################################################################
43
60
 
44
- Style/BracesAroundHashParameters:
45
- Enabled: false
46
-
47
61
  Style/Documentation:
48
62
  Enabled: false
49
63
 
64
+ Style/ExponentialNotation:
65
+ Enabled: true
66
+
67
+ Style/HashEachMethods:
68
+ Enabled: true
69
+
50
70
  Style/HashSyntax:
51
71
  EnforcedStyle: hash_rockets
52
72
 
73
+ Style/HashTransformKeys:
74
+ Enabled: true
75
+
76
+ Style/HashTransformValues:
77
+ Enabled: true
78
+
53
79
  # Follow your heart where it makes sense to use lambda or lambda literal.
54
80
  # Enforcing it makes some pieces of code look REALLY terrible, e.g. in
55
81
  # case of empty (noop) lambdas: `lambda { |_| }`.
@@ -1,5 +1,4 @@
1
1
  language: ruby
2
- sudo: false
3
2
 
4
3
  services:
5
4
  - redis-server
@@ -9,8 +8,9 @@ cache: bundler
9
8
  before_install:
10
9
  - gem update --system
11
10
  - gem --version
12
- - gem install bundler --no-rdoc --no-ri
11
+ - gem install bundler
13
12
  - bundle --version
13
+ # Install pahantomjs
14
14
  - mkdir travis-phantomjs
15
15
  - wget https://s3.amazonaws.com/travis-phantomjs/phantomjs-2.0.0-ubuntu-12.04.tar.bz2 -O $PWD/travis-phantomjs/phantomjs-2.0.0-ubuntu-12.04.tar.bz2
16
16
  - tar -xvf $PWD/travis-phantomjs/phantomjs-2.0.0-ubuntu-12.04.tar.bz2 -C $PWD/travis-phantomjs
@@ -19,15 +19,16 @@ before_install:
19
19
  install: bundle install --without development
20
20
 
21
21
  rvm:
22
- - 2.3
23
22
  - 2.4
24
23
  - 2.5
24
+ - 2.6
25
+ - 2.7
25
26
 
26
27
  matrix:
27
28
  fast_finish: true
28
29
  include:
29
30
  -
30
- rvm: 2.3
31
+ rvm: 2.4
31
32
  env: SUITE="rubocop"
32
33
  gemfile: Gemfile
33
34
 
@@ -35,3 +36,4 @@ gemfile:
35
36
  - gemfiles/sidekiq_5.0.gemfile
36
37
  - gemfiles/sidekiq_5.1.gemfile
37
38
  - gemfiles/sidekiq_5.2.gemfile
39
+ - gemfiles/sidekiq_6.0.gemfile
data/Appraisals CHANGED
@@ -11,3 +11,11 @@ end
11
11
  appraise "sidekiq-5.2" do
12
12
  gem "sidekiq", "~> 5.2.0"
13
13
  end
14
+
15
+ appraise "sidekiq-6.0" do
16
+ gem "sidekiq", "~> 6.0.0"
17
+ end
18
+
19
+ appraise "sidekiq-6.1" do
20
+ gem "sidekiq", "~> 6.1.0"
21
+ end
data/CHANGES.md CHANGED
@@ -1,3 +1,42 @@
1
+ ## 0.13.0 (2020-07-28)
2
+
3
+ * [#85](https://github.com/sensortower/sidekiq-throttled/pull/85)
4
+ Add Sidekiq 6.1+ support.
5
+ ([@hmaack])
6
+
7
+ ## 0.12.0 (2020-06-22)
8
+
9
+ * [#80](https://github.com/sensortower/sidekiq-throttled/pull/80)
10
+ Allow override cooldown timeout of queues with throttled jobs.
11
+ ([@vaot])
12
+
13
+ * [#76](https://github.com/sensortower/sidekiq-throttled/pull/76)
14
+ Fix warnings on Ruby 2.7
15
+ ([@lenon])
16
+
17
+
18
+ ## 0.11.0 (2019-08-24)
19
+
20
+ * [#59](https://github.com/sensortower/sidekiq-throttled/pull/59)
21
+ Add throttling observer.
22
+ ([@ogins57])
23
+
24
+
25
+ ## 0.10.0 (2019-06-22)
26
+
27
+ * [#60](https://github.com/sensortower/sidekiq-throttled/pull/60)
28
+ Skip throttling check in redis if limit is 0.
29
+ ([@mstruve])
30
+
31
+ * [#58](https://github.com/sensortower/sidekiq-throttled/pull/58)
32
+ Improve documentation bout TTL.
33
+ ([@ziaulrehman40])
34
+
35
+ * Improve reliability of paused queues, by resyncing list of paused queues
36
+ on schedule.
37
+ ([@ixti])
38
+
39
+
1
40
  ## 0.9.0 (2018-09-11)
2
41
 
3
42
  * Add support of Sidekiq 5.2.x
@@ -176,3 +215,9 @@
176
215
  [@palanglung]: https://github.com/palanglung
177
216
  [@azach]: https://github.com/azach
178
217
  [@iporsut]: https://github.com/iporsut
218
+ [@mstruve]: https://github.com/mstruve
219
+ [@ziaulrehman40]: https://github.com/ziaulrehman40
220
+ [@ogins57]: https://github.com/ogins57
221
+ [@lenon]: https://github.com/lenon
222
+ [@vaot]: https://github.com/vaot
223
+ [@hmaack]: https://github.com/hmaack
data/Gemfile CHANGED
@@ -5,8 +5,9 @@ source "https://rubygems.org"
5
5
  gem "appraisal"
6
6
  gem "rake"
7
7
  gem "rspec"
8
- gem "rubocop", "~> 0.58.0", :require => false
9
- gem "rubocop-rspec", "~> 1.29.1", :require => false
8
+ gem "rubocop", "~> 0.82.0", :require => false
9
+ gem "rubocop-performance", "~>1.5.2", :require => false
10
+ gem "rubocop-rspec", "~> 1.39.0", :require => false
10
11
  gem "sidekiq"
11
12
 
12
13
  group :development do
@@ -17,13 +18,13 @@ group :development do
17
18
  end
18
19
 
19
20
  group :test do
21
+ gem "apparition"
20
22
  gem "capybara"
21
23
  gem "coveralls", :require => false
22
- gem "poltergeist"
23
24
  gem "puma"
24
25
  gem "rack-test"
25
- gem "simplecov", ">= 0.9"
26
- gem "sinatra", "~> 1.4", ">= 1.4.6"
26
+ gem "simplecov"
27
+ gem "sinatra"
27
28
  gem "timecop"
28
29
  end
29
30
 
data/LICENSE.md CHANGED
@@ -1,6 +1,6 @@
1
1
  The MIT License (MIT)
2
2
 
3
- Copyright (c) 2015-2018 SensorTower Inc.
3
+ Copyright (c) 2015-2020 SensorTower Inc.
4
4
 
5
5
  Permission is hereby granted, free of charge, to any person obtaining a copy
6
6
  of this software and associated documentation files (the "Software"), to deal
data/README.md CHANGED
@@ -1,14 +1,14 @@
1
1
  # Sidekiq::Throttled
2
2
 
3
- [![Gem Version](https://badge.fury.io/rb/sidekiq-throttled.svg)](http://rubygems.org/gems/sidekiq-throttled)
4
- [![Build Status](https://travis-ci.org/sensortower/sidekiq-throttled.svg?branch=master)](https://travis-ci.org/sensortower/sidekiq-throttled)
5
- [![Code Climate](https://codeclimate.com/github/sensortower/sidekiq-throttled.svg?branch=master)](https://codeclimate.com/github/sensortower/sidekiq-throttled)
6
- [![Coverage Status](https://coveralls.io/repos/github/sensortower/sidekiq-throttled/badge.svg?branch=master)](https://coveralls.io/github/sensortower/sidekiq-throttled?branch=master)
7
- [![API Docs](http://inch-ci.org/github/sensortower/sidekiq-throttled.svg?branch=master)](http://inch-ci.org/github/sensortower/sidekiq-throttled)
3
+ [![Latest Version](https://badge.fury.io/rb/sidekiq-throttled.svg)](http://rubygems.org/gems/sidekiq-throttled)
4
+ [![CI Status](https://github.com/sensortower/sidekiq-throttled/workflows/CI/badge.svg?branch=master)](https://github.com/sensortower/sidekiq-throttled/actions?query=workflow%3ACI+branch%3Amaster)
5
+ [![Code Quality](https://codeclimate.com/github/sensortower/sidekiq-throttled.svg?branch=master)](https://codeclimate.com/github/sensortower/sidekiq-throttled)
6
+ [![Code Coverage](https://coveralls.io/repos/github/sensortower/sidekiq-throttled/badge.svg?branch=master)](https://coveralls.io/github/sensortower/sidekiq-throttled?branch=master)
7
+ [![API Docs Quality](http://inch-ci.org/github/sensortower/sidekiq-throttled.svg?branch=master)](http://inch-ci.org/github/sensortower/sidekiq-throttled)
8
+ [![API Docs](https://img.shields.io/badge/yard-docs-blue.svg)](http://www.rubydoc.info/gems/sidekiq-throttled)
8
9
 
9
10
  Concurrency and threshold throttling for [Sidekiq][sidekiq].
10
11
 
11
-
12
12
  ## Installation
13
13
 
14
14
  Add this line to your application's Gemfile:
@@ -36,6 +36,9 @@ require "sidekiq/throttled"
36
36
  Sidekiq::Throttled.setup!
37
37
  ```
38
38
 
39
+ Load order can be an issue if you are using other Sidekiq plugins and/or middleware.
40
+ To prevent any problems, add the `.setup!` call to the bottom of your init file.
41
+
39
42
  Once you've done that you can include `Sidekiq::Throttled::Worker` to your
40
43
  job classes and configure throttling:
41
44
 
@@ -59,6 +62,38 @@ class MyWorker
59
62
  end
60
63
  ```
61
64
 
65
+ ### Observer
66
+
67
+ You can specify an observer that will be called on throttling. To do so pass an
68
+ `:observer` option with callable object:
69
+
70
+ ``` ruby
71
+ class MyWorker
72
+ include Sidekiq::Worker
73
+ include Sidekiq::Throttled::Worker
74
+
75
+ MY_OBSERVER = lambda do |strategy, *args|
76
+ # do something
77
+ end
78
+
79
+ sidekiq_options :queue => :my_queue
80
+
81
+ sidekiq_throttle({
82
+ :concurrency => { :limit => 10 },
83
+ :threshold => { :limit => 100, :period => 1.hour }
84
+ :observer => MY_OBSERVER
85
+ })
86
+
87
+ def perform(*args)
88
+ # ...
89
+ end
90
+ end
91
+ ```
92
+
93
+ Observer will receive `strategy, *args` arguments, where `strategy` is a Symbol
94
+ `:concurrency` or `:threshold`, and `*args` are the arguments that were passed
95
+ to the job.
96
+
62
97
 
63
98
  ### Dynamic throttling
64
99
 
@@ -117,6 +152,25 @@ if you are using dynamic limit/period options. Otherwise you risk getting into
117
152
  some trouble.
118
153
 
119
154
 
155
+ ### Concurrency throttling fine-tuning
156
+
157
+ Concurrency throttling is based on distributed locks. Those locks have default
158
+ time to live (TTL) set to 15 minutes. If your job takes more than 15 minutes
159
+ to finish, lock will be released and you might end up with more jobs running
160
+ concurrently than you expect.
161
+
162
+ This is done to avoid deadlocks - when by any reason (e.g. Sidekiq process was
163
+ OOM-killed) cleanup middleware wasn't executed and locks were not released.
164
+
165
+ If your job takes more than 15 minutes to complete, you can tune concurrency
166
+ lock TTL to fit your needs:
167
+
168
+ ``` ruby
169
+ # Set concurrency strategy lock TTL to 1 hour.
170
+ sidekiq_throttle(:concurrency => { :limit => 20, :ttl => 1.hour.to_i })
171
+ ```
172
+
173
+
120
174
  ## Enhanced Queues list
121
175
 
122
176
  This gem provides ability to pause/resume queues from processing by workers.
@@ -155,9 +209,9 @@ end
155
209
  This library aims to support and is [tested against][travis] the following Ruby
156
210
  versions:
157
211
 
158
- * Ruby 2.3.x
159
212
  * Ruby 2.4.x
160
213
  * Ruby 2.5.x
214
+ * Ruby 2.6.x
161
215
 
162
216
  If something doesn't work on one of these versions, it's a bug.
163
217
 
@@ -180,6 +234,8 @@ This library aims to support work with following [Sidekiq][sidekiq] versions:
180
234
  * Sidekiq 5.0.x
181
235
  * Sidekiq 5.1.x
182
236
  * Sidekiq 5.2.x
237
+ * Sidekiq 6.0.x
238
+ * Sidekiq 6.1.x
183
239
 
184
240
 
185
241
  ## Contributing
@@ -207,7 +263,7 @@ Don't forget to run `appraisal update` after any changes to `Gemfile`.
207
263
 
208
264
  ## Copyright
209
265
 
210
- Copyright (c) 2015-2018 SensorTower Inc.
266
+ Copyright (c) 2015-2020 SensorTower Inc.
211
267
  See LICENSE.md for further details.
212
268
 
213
269
 
@@ -5,8 +5,9 @@ source "https://rubygems.org"
5
5
  gem "appraisal"
6
6
  gem "rake"
7
7
  gem "rspec"
8
- gem "rubocop", "~> 0.58.0", require: false
9
- gem "rubocop-rspec", "~> 1.29.1", require: false
8
+ gem "rubocop", "~> 0.82.0", require: false
9
+ gem "rubocop-performance", "~>1.5.2", require: false
10
+ gem "rubocop-rspec", "~> 1.39.0", require: false
10
11
  gem "sidekiq", "~> 5.0.0"
11
12
 
12
13
  group :development do
@@ -17,13 +18,13 @@ group :development do
17
18
  end
18
19
 
19
20
  group :test do
21
+ gem "apparition"
20
22
  gem "capybara"
21
23
  gem "coveralls", require: false
22
- gem "poltergeist"
23
24
  gem "puma"
24
25
  gem "rack-test"
25
- gem "simplecov", ">= 0.9"
26
- gem "sinatra", "~> 1.4", ">= 1.4.6"
26
+ gem "simplecov"
27
+ gem "sinatra"
27
28
  gem "timecop"
28
29
  end
29
30
 
@@ -5,8 +5,9 @@ source "https://rubygems.org"
5
5
  gem "appraisal"
6
6
  gem "rake"
7
7
  gem "rspec"
8
- gem "rubocop", "~> 0.58.0", require: false
9
- gem "rubocop-rspec", "~> 1.29.1", require: false
8
+ gem "rubocop", "~> 0.82.0", require: false
9
+ gem "rubocop-performance", "~>1.5.2", require: false
10
+ gem "rubocop-rspec", "~> 1.39.0", require: false
10
11
  gem "sidekiq", "~> 5.1.0"
11
12
 
12
13
  group :development do
@@ -17,13 +18,13 @@ group :development do
17
18
  end
18
19
 
19
20
  group :test do
21
+ gem "apparition"
20
22
  gem "capybara"
21
23
  gem "coveralls", require: false
22
- gem "poltergeist"
23
24
  gem "puma"
24
25
  gem "rack-test"
25
- gem "simplecov", ">= 0.9"
26
- gem "sinatra", "~> 1.4", ">= 1.4.6"
26
+ gem "simplecov"
27
+ gem "sinatra"
27
28
  gem "timecop"
28
29
  end
29
30
 
@@ -5,8 +5,9 @@ source "https://rubygems.org"
5
5
  gem "appraisal"
6
6
  gem "rake"
7
7
  gem "rspec"
8
- gem "rubocop", "~> 0.58.0", require: false
9
- gem "rubocop-rspec", "~> 1.29.1", require: false
8
+ gem "rubocop", "~> 0.82.0", require: false
9
+ gem "rubocop-performance", "~>1.5.2", require: false
10
+ gem "rubocop-rspec", "~> 1.39.0", require: false
10
11
  gem "sidekiq", "~> 5.2.0"
11
12
 
12
13
  group :development do
@@ -17,13 +18,13 @@ group :development do
17
18
  end
18
19
 
19
20
  group :test do
21
+ gem "apparition"
20
22
  gem "capybara"
21
23
  gem "coveralls", require: false
22
- gem "poltergeist"
23
24
  gem "puma"
24
25
  gem "rack-test"
25
- gem "simplecov", ">= 0.9"
26
- gem "sinatra", "~> 1.4", ">= 1.4.6"
26
+ gem "simplecov"
27
+ gem "sinatra"
27
28
  gem "timecop"
28
29
  end
29
30
 
@@ -0,0 +1,31 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "appraisal"
6
+ gem "rake"
7
+ gem "rspec"
8
+ gem "rubocop", "~> 0.82.0", require: false
9
+ gem "rubocop-performance", "~>1.5.2", require: false
10
+ gem "rubocop-rspec", "~> 1.39.0", require: false
11
+ gem "sidekiq", "~> 6.0.0"
12
+
13
+ group :development do
14
+ gem "byebug"
15
+ gem "guard", require: false
16
+ gem "guard-rspec", require: false
17
+ gem "guard-rubocop", require: false
18
+ end
19
+
20
+ group :test do
21
+ gem "apparition"
22
+ gem "capybara"
23
+ gem "coveralls", require: false
24
+ gem "puma"
25
+ gem "rack-test"
26
+ gem "simplecov"
27
+ gem "sinatra"
28
+ gem "timecop"
29
+ end
30
+
31
+ gemspec path: "../"
@@ -0,0 +1,31 @@
1
+ # This file was generated by Appraisal
2
+
3
+ source "https://rubygems.org"
4
+
5
+ gem "appraisal"
6
+ gem "rake"
7
+ gem "rspec"
8
+ gem "rubocop", "~> 0.82.0", require: false
9
+ gem "rubocop-performance", "~>1.5.2", require: false
10
+ gem "rubocop-rspec", "~> 1.39.0", require: false
11
+ gem "sidekiq", "~> 6.1.0"
12
+
13
+ group :development do
14
+ gem "byebug"
15
+ gem "guard", require: false
16
+ gem "guard-rspec", require: false
17
+ gem "guard-rubocop", require: false
18
+ end
19
+
20
+ group :test do
21
+ gem "apparition"
22
+ gem "capybara"
23
+ gem "coveralls", require: false
24
+ gem "puma"
25
+ gem "rack-test"
26
+ gem "simplecov"
27
+ gem "sinatra"
28
+ gem "timecop"
29
+ end
30
+
31
+ gemspec path: "../"
@@ -6,9 +6,11 @@ require "sidekiq"
6
6
  # internal
7
7
  require "sidekiq/throttled/version"
8
8
  require "sidekiq/throttled/communicator"
9
+ require "sidekiq/throttled/configuration"
9
10
  require "sidekiq/throttled/queues_pauser"
10
11
  require "sidekiq/throttled/registry"
11
12
  require "sidekiq/throttled/worker"
13
+ require "sidekiq/throttled/utils"
12
14
 
13
15
  # @see https://github.com/mperham/sidekiq/
14
16
  module Sidekiq
@@ -44,6 +46,13 @@ module Sidekiq
44
46
  private_constant :MUTEX
45
47
 
46
48
  class << self
49
+ include Utils
50
+
51
+ # @return [Configuration]
52
+ def configuration
53
+ @configuration ||= Configuration.new
54
+ end
55
+
47
56
  # Hooks throttler into sidekiq.
48
57
  #
49
58
  # @return [void]
@@ -52,8 +61,7 @@ module Sidekiq
52
61
  QueuesPauser.instance.setup!
53
62
 
54
63
  Sidekiq.configure_server do |config|
55
- require "sidekiq/throttled/fetch"
56
- Sidekiq.options[:fetch] = Sidekiq::Throttled::Fetch
64
+ setup_strategy!
57
65
 
58
66
  require "sidekiq/throttled/middleware"
59
67
  config.server_middleware do |chain|
@@ -84,6 +92,19 @@ module Sidekiq
84
92
 
85
93
  private
86
94
 
95
+ # @return [void]
96
+ def setup_strategy!
97
+ require "sidekiq/throttled/fetch"
98
+
99
+ # https://github.com/mperham/sidekiq/commit/fce05c9d4b4c0411c982078a4cf3a63f20f739bc
100
+ Sidekiq.options[:fetch] =
101
+ if Gem::Version.new(Sidekiq::VERSION) < Gem::Version.new("6.1.0")
102
+ Sidekiq::Throttled::Fetch
103
+ else
104
+ Sidekiq::Throttled::Fetch.new(Sidekiq.options)
105
+ end
106
+ end
107
+
87
108
  # Tries to preload constant by it's name once.
88
109
  #
89
110
  # Somehow, sometimes, some classes are not eager loaded upon Rails init,
@@ -99,16 +120,6 @@ module Sidekiq
99
120
  @preloaded[job] ||= constantize(job) || true
100
121
  end
101
122
  end
102
-
103
- # Resolve constant from it's name
104
- def constantize(str)
105
- str.sub(/^::/, "").split("::").inject(Object) do |const, name|
106
- const.const_get(name)
107
- end
108
- rescue
109
- Sidekiq.logger.warn { "Failed to constantize: #{str}" }
110
- nil
111
- end
112
123
  end
113
124
  end
114
125
  end
@@ -43,6 +43,7 @@ module Sidekiq
43
43
  # @return [self]
44
44
  def on(event, &handler)
45
45
  raise ArgumentError, "No block given" unless handler
46
+
46
47
  @mutex.synchronize { @handlers[event.to_s] << handler }
47
48
  self
48
49
  end
@@ -0,0 +1,50 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Sidekiq
4
+ module Throttled
5
+ # Configuration holder.
6
+ class Configuration
7
+ # Class constructor.
8
+ def initialize
9
+ reset!
10
+ end
11
+
12
+ # Reset configuration to defaults.
13
+ #
14
+ # @return [self]
15
+ def reset!
16
+ @inherit_strategies = false
17
+
18
+ self
19
+ end
20
+
21
+ # Instructs throttler to lookup strategies in parent classes, if there's
22
+ # no own strategy:
23
+ #
24
+ # class Foo
25
+ # include Sidekiq::Worker
26
+ # include Sidekiq::Worker::Throttled
27
+ #
28
+ # sidekiq_throttle :concurrency => { :limit => 42 }
29
+ # end
30
+ #
31
+ # class Bar < Foo
32
+ # end
33
+ #
34
+ # By default in the example above, `Bar` won't have throttling options.
35
+ # Set this flag to `true` to enable this lookup in initializer, after
36
+ # that `Bar` will use `Foo` throttling bucket.
37
+ def inherit_strategies=(value)
38
+ @inherit_strategies = value ? true : false
39
+ end
40
+
41
+ # Whenever throttled workers should inherit parent's strategies or not.
42
+ # Default: `false`.
43
+ #
44
+ # @return [Boolean]
45
+ def inherit_strategies?
46
+ @inherit_strategies
47
+ end
48
+ end
49
+ end
50
+ end
@@ -11,7 +11,7 @@ module Sidekiq
11
11
  #
12
12
  # ## Implementation
13
13
  #
14
- # Internally list holds an array of arrays. Thus ecah element is a tuple of
14
+ # Internally list holds an array of arrays. Thus each element is a tuple of
15
15
  # monotonic timestamp (when element was added) and element itself:
16
16
  #
17
17
  # [
@@ -12,16 +12,47 @@ module Sidekiq
12
12
  #
13
13
  # @private
14
14
  class Fetch
15
+ module BulkRequeue
16
+ # Requeues all given units as a single operation.
17
+ #
18
+ # @see http://www.rubydoc.info/github/redis/redis-rb/master/Redis#pipelined-instance_method
19
+ # @param [Array<Fetch::UnitOfWork>] units
20
+ # @return [void]
21
+ def bulk_requeue(units, _options)
22
+ return if units.empty?
23
+
24
+ Sidekiq.logger.debug { "Re-queueing terminated jobs" }
25
+ Sidekiq.redis { |conn| conn.pipelined { units.each(&:requeue) } }
26
+ Sidekiq.logger.info("Pushed #{units.size} jobs back to Redis")
27
+ rescue => e
28
+ Sidekiq.logger.warn("Failed to requeue #{units.size} jobs: #{e}")
29
+ end
30
+ end
31
+
32
+ # https://github.com/mperham/sidekiq/commit/fce05c9d4b4c0411c982078a4cf3a63f20f739bc
33
+ if Gem::Version.new(Sidekiq::VERSION) < Gem::Version.new("6.1.0")
34
+ extend BulkRequeue
35
+ else
36
+ include BulkRequeue
37
+ end
15
38
  # Timeout to sleep between fetch retries in case of no job received,
16
39
  # as well as timeout to wait for redis to give us something to work.
17
40
  TIMEOUT = 2
18
41
 
19
42
  # Initializes fetcher instance.
43
+ # @param options [Hash]
44
+ # @option options [Integer] :throttled_queue_cooldown (TIMEOUT)
45
+ # Min delay in seconds before queue will be polled again after
46
+ # throttled job.
47
+ # @option options [Boolean] :strict (false)
48
+ # @option options [Array<#to_s>] :queue
20
49
  def initialize(options)
21
- @paused = ExpirableList.new(TIMEOUT)
50
+ @paused = ExpirableList.new(options.fetch(:throttled_queue_cooldown, TIMEOUT))
22
51
 
23
- @strict = options[:strict]
24
- @queues = options[:queues].map { |q| QueueName.expand q }
52
+ @strict = options.fetch(:strict, false)
53
+ @queues = options.fetch(:queues).map { |q| QueueName.expand q }
54
+
55
+ raise ArgumentError, "empty :queues" if @queues.empty?
25
56
 
26
57
  @queues.uniq! if @strict
27
58
  end
@@ -42,23 +73,6 @@ module Sidekiq
42
73
  nil
43
74
  end
44
75
 
45
- class << self
46
- # Requeues all given units as a single operation.
47
- #
48
- # @see http://www.rubydoc.info/github/redis/redis-rb/master/Redis#pipelined-instance_method
49
- # @param [Array<Fetch::UnitOfWork>] units
50
- # @return [void]
51
- def bulk_requeue(units, _options)
52
- return if units.empty?
53
-
54
- Sidekiq.logger.debug { "Re-queueing terminated jobs" }
55
- Sidekiq.redis { |conn| conn.pipelined { units.each(&:requeue) } }
56
- Sidekiq.logger.info("Pushed #{units.size} jobs back to Redis")
57
- rescue => e
58
- Sidekiq.logger.warn("Failed to requeue #{units.size} jobs: #{e}")
59
- end
60
- end
61
-
62
76
  private
63
77
 
64
78
  # Tries to pop pair of `queue` and job `message` out of sidekiq queues.
@@ -7,7 +7,7 @@ module Sidekiq
7
7
  # @private
8
8
  module QueueName
9
9
  # RegExp used to stip out any redisr-namespace prefixes with `queue:`.
10
- QUEUE_NAME_PREFIX_RE = /.*queue:/
10
+ QUEUE_NAME_PREFIX_RE = /.*queue:/.freeze
11
11
  private_constant :QUEUE_NAME_PREFIX_RE
12
12
 
13
13
  class << self
@@ -27,7 +27,7 @@ module Sidekiq
27
27
  # @param [#to_s]
28
28
  # @return [String]
29
29
  def normalize(queue)
30
- queue.to_s.sub(QUEUE_NAME_PREFIX_RE, "")
30
+ -queue.to_s.sub(QUEUE_NAME_PREFIX_RE, "")
31
31
  end
32
32
 
33
33
  # Prepends `queue:` prefix to given `queue` name.
@@ -38,7 +38,7 @@ module Sidekiq
38
38
  # @param [#to_s] queue Queue name
39
39
  # @return [String]
40
40
  def expand(queue)
41
- "queue:#{queue}"
41
+ -"queue:#{queue}"
42
42
  end
43
43
  end
44
44
  end
@@ -2,6 +2,7 @@
2
2
 
3
3
  require "set"
4
4
  require "singleton"
5
+ require "concurrent/timer_task"
5
6
 
6
7
  require "sidekiq/throttled/patches/queue"
7
8
  require "sidekiq/throttled/communicator"
@@ -55,7 +56,7 @@ module Sidekiq
55
56
 
56
57
  @communicator.receive(PAUSE_MESSAGE, &method(:add))
57
58
  @communicator.receive(RESUME_MESSAGE, &method(:delete))
58
- @communicator.ready(&method(:sync!))
59
+ @communicator.ready { sync! }
59
60
  end
60
61
  end
61
62
 
@@ -64,7 +65,10 @@ module Sidekiq
64
65
  # @private
65
66
  # @return [Array<String>]
66
67
  def filter(queues)
67
- queues - @paused_queues.to_a
68
+ @mutex.synchronize { queues - @paused_queues.to_a }
69
+ rescue => e
70
+ Sidekiq.logger.error { "[#{self.class}] Failed filter queues: #{e}" }
71
+ queues
68
72
  end
69
73
 
70
74
  # Returns list of paused queues.
@@ -2,6 +2,7 @@
2
2
 
3
3
  # internal
4
4
  require "sidekiq/throttled/strategy"
5
+ require "sidekiq/throttled/utils"
5
6
 
6
7
  module Sidekiq
7
8
  module Throttled
@@ -13,6 +14,8 @@ module Sidekiq
13
14
  @aliases = {}
14
15
 
15
16
  class << self
17
+ include Utils
18
+
16
19
  # Adds strategy to the registry.
17
20
  #
18
21
  # @note prints a warning to STDERR upon duplicate strategy name
@@ -49,12 +52,15 @@ module Sidekiq
49
52
  #
50
53
  # @overload get(name, &block)
51
54
  # Yields control to the block if requested strategy was found.
55
+ # @param [#to_s] name
52
56
  # @yieldparam [Strategy] strategy
53
57
  # @yield [strategy] Gives found strategy to the block
54
58
  # @return result of a block
55
59
  def get(name)
56
- strategy = @strategies[name.to_s] || @aliases[name.to_s]
60
+ strategy = find(name.to_s) || find_by_class(name)
61
+
57
62
  return yield strategy if strategy && block_given?
63
+
58
64
  strategy
59
65
  end
60
66
 
@@ -68,6 +74,7 @@ module Sidekiq
68
74
  # @return [Registry]
69
75
  def each
70
76
  return to_enum(__method__) unless block_given?
77
+
71
78
  @strategies.each { |*args| yield(*args) }
72
79
  self
73
80
  end
@@ -82,10 +89,39 @@ module Sidekiq
82
89
  # @return [Registry]
83
90
  def each_with_static_keys
84
91
  return to_enum(__method__) unless block_given?
92
+
85
93
  @strategies.each do |name, strategy|
86
94
  yield(name, strategy) unless strategy.dynamic?
87
95
  end
88
96
  end
97
+
98
+ private
99
+
100
+ # Find strategy by it's name.
101
+ #
102
+ # @param name [String]
103
+ # @return [Strategy, nil]
104
+ def find(name)
105
+ @strategies[name] || @aliases[name]
106
+ end
107
+
108
+ # Find strategy by class or it's parents.
109
+ #
110
+ # @param name [Class, #to_s]
111
+ # @return [Strategy, nil]
112
+ def find_by_class(name)
113
+ return unless Throttled.configuration.inherit_strategies?
114
+
115
+ const = name.is_a?(Class) ? name : constantize(name)
116
+ return unless const.is_a?(Class)
117
+
118
+ const.ancestors.each do |m|
119
+ strategy = find(m.name)
120
+ return strategy if strategy
121
+ end
122
+
123
+ nil
124
+ end
89
125
  end
90
126
  end
91
127
  end
@@ -19,26 +19,24 @@ module Sidekiq
19
19
  # @return [Strategy::Threshold, nil]
20
20
  attr_reader :threshold
21
21
 
22
+ # @!attribute [r] observer
23
+ # @return [Proc, nil]
24
+ attr_reader :observer
25
+
22
26
  # @param [#to_s] name
23
27
  # @param [Hash] concurrency Concurrency options.
24
28
  # See keyword args of {Strategy::Concurrency#initialize} for details.
25
29
  # @param [Hash] threshold Threshold options.
26
30
  # See keyword args of {Strategy::Threshold#initialize} for details.
27
31
  # @param [#call] key_suffix Dynamic key suffix generator.
28
- def initialize(name, concurrency: nil, threshold: nil, key_suffix: nil)
29
- key = "throttled:#{name}"
30
-
31
- @concurrency =
32
- if concurrency
33
- concurrency[:key_suffix] ||= key_suffix
34
- Concurrency.new(key, **concurrency)
35
- end
36
-
37
- @threshold =
38
- if threshold
39
- threshold[:key_suffix] ||= key_suffix
40
- Threshold.new(key, **threshold)
41
- end
32
+ # @param [#call] observer Process called after throttled.
33
+ def initialize(
34
+ name,
35
+ concurrency: nil, threshold: nil, key_suffix: nil, observer: nil
36
+ )
37
+ @observer = observer
38
+ @concurrency = make_strategy(Concurrency, name, key_suffix, concurrency)
39
+ @threshold = make_strategy(Threshold, name, key_suffix, threshold)
42
40
 
43
41
  return if @concurrency || @threshold
44
42
 
@@ -55,9 +53,13 @@ module Sidekiq
55
53
 
56
54
  # @return [Boolean] whenever job is throttled or not.
57
55
  def throttled?(jid, *job_args)
58
- return true if @concurrency&.throttled?(jid, *job_args)
56
+ if @concurrency&.throttled?(jid, *job_args)
57
+ @observer&.call(:concurrency, *job_args)
58
+ return true
59
+ end
59
60
 
60
61
  if @threshold&.throttled?(*job_args)
62
+ @observer&.call(:threshold, *job_args)
61
63
  finalize!(jid, *job_args)
62
64
  return true
63
65
  end
@@ -77,6 +79,15 @@ module Sidekiq
77
79
  @concurrency&.reset!
78
80
  @threshold&.reset!
79
81
  end
82
+
83
+ private
84
+
85
+ # @return [Base, nil]
86
+ def make_strategy(strategy, name, key_suffix, options)
87
+ return unless options
88
+
89
+ strategy.new("throttled:#{name}", :key_suffix => key_suffix, **options)
90
+ end
80
91
  end
81
92
  end
82
93
  end
@@ -42,14 +42,16 @@ module Sidekiq
42
42
 
43
43
  # @return [Boolean] whenever job is throttled or not
44
44
  def throttled?(jid, *job_args)
45
- return false unless (job_limit = limit(job_args))
45
+ job_limit = limit(job_args)
46
+ return false unless job_limit
47
+ return true if job_limit <= 0
46
48
 
47
- kwargs = {
48
- :keys => [key(job_args)],
49
- :argv => [jid.to_s, job_limit, @ttl, Time.now.to_f]
50
- }
49
+ keys = [key(job_args)]
50
+ argv = [jid.to_s, job_limit, @ttl, Time.now.to_f]
51
51
 
52
- Sidekiq.redis { |redis| 1 == SCRIPT.eval(redis, kwargs) }
52
+ Sidekiq.redis do |redis|
53
+ 1 == SCRIPT.eval(redis, :keys => keys, :argv => argv)
54
+ end
53
55
  end
54
56
 
55
57
  # @return [Integer] Current count of jobs
@@ -48,6 +48,7 @@ module Sidekiq
48
48
  # @return [Float] Period in seconds
49
49
  def period(job_args = nil)
50
50
  return @period.to_f unless @period.respond_to? :call
51
+
51
52
  @period.call(*job_args).to_f
52
53
  end
53
54
 
@@ -58,14 +59,16 @@ module Sidekiq
58
59
 
59
60
  # @return [Boolean] whenever job is throttled or not
60
61
  def throttled?(*job_args)
61
- return false unless (job_limit = limit(job_args))
62
+ job_limit = limit(job_args)
63
+ return false unless job_limit
64
+ return true if job_limit <= 0
62
65
 
63
- kwargs = {
64
- :keys => [key(job_args)],
65
- :argv => [job_limit, period(job_args), Time.now.to_f]
66
- }
66
+ keys = [key(job_args)]
67
+ argv = [job_limit, period(job_args), Time.now.to_f]
67
68
 
68
- Sidekiq.redis { |redis| 1 == SCRIPT.eval(redis, kwargs) }
69
+ Sidekiq.redis do |redis|
70
+ 1 == SCRIPT.eval(redis, :keys => keys, :argv => argv)
71
+ end
69
72
  end
70
73
 
71
74
  # @return [Integer] Current count of jobs
@@ -0,0 +1,19 @@
1
+ # frozen_string_literal: true
2
+
3
+ module Sidekiq
4
+ module Throttled
5
+ module Utils
6
+ module_function
7
+
8
+ # Resolve constant from it's name
9
+ # @param name [#to_s] Constant name
10
+ # @return [Object, nil] Resolved constant or nil if failed.
11
+ def constantize(name)
12
+ name.to_s.sub(/^::/, "").split("::").inject(Object, &:const_get)
13
+ rescue NameError
14
+ Sidekiq.logger.warn { "Failed to constantize: #{name}" }
15
+ nil
16
+ end
17
+ end
18
+ end
19
+ end
@@ -3,6 +3,6 @@
3
3
  module Sidekiq
4
4
  module Throttled
5
5
  # Gem version
6
- VERSION = "0.10.0.alpha"
6
+ VERSION = "0.13.0"
7
7
  end
8
8
  end
@@ -14,9 +14,7 @@ module Sidekiq
14
14
 
15
15
  # @param [Strategy::Concurrency, Strategy::Threshold] strategy
16
16
  def initialize(strategy)
17
- if strategy&.dynamic?
18
- raise ArgumentError, "Can't handle dynamic strategies"
19
- end
17
+ raise ArgumentError, "Can't handle dynamic strategies" if strategy&.dynamic?
20
18
 
21
19
  @strategy = strategy
22
20
  end
@@ -27,9 +25,7 @@ module Sidekiq
27
25
 
28
26
  html = humanize_integer(@strategy.limit) << " jobs"
29
27
 
30
- if @strategy.respond_to? :period
31
- html << " per " << humanize_duration(@strategy.period)
32
- end
28
+ html << " per " << humanize_duration(@strategy.period) if @strategy.respond_to?(:period)
33
29
 
34
30
  html << "<br />" << colorize_count(@strategy.count, @strategy.limit)
35
31
  end
@@ -11,7 +11,8 @@ module Sidekiq
11
11
  attr_accessor :enabled
12
12
 
13
13
  def apply!(app)
14
- Sidekiq::WebAction.send(:prepend, SummaryFix)
14
+ Sidekiq::WebAction.prepend SummaryFix
15
+
15
16
  app.get("/throttled/summary_fix") do
16
17
  [200, HEADERS.dup, JAVASCRIPT.dup]
17
18
  end
@@ -28,5 +28,5 @@ Gem::Specification.new do |spec|
28
28
  spec.add_runtime_dependency "redis-prescription"
29
29
  spec.add_runtime_dependency "sidekiq"
30
30
 
31
- spec.add_development_dependency "bundler", "~> 1.10"
31
+ spec.add_development_dependency "bundler", "~> 2.0"
32
32
  end
metadata CHANGED
@@ -1,14 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sidekiq-throttled
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.10.0.alpha
4
+ version: 0.13.0
5
5
  platform: ruby
6
6
  authors:
7
7
  - Alexey V Zapparov
8
8
  autorequire:
9
9
  bindir: exe
10
10
  cert_chain: []
11
- date: 2018-09-21 00:00:00.000000000 Z
11
+ date: 2020-07-28 00:00:00.000000000 Z
12
12
  dependencies:
13
13
  - !ruby/object:Gem::Dependency
14
14
  name: concurrent-ruby
@@ -58,14 +58,14 @@ dependencies:
58
58
  requirements:
59
59
  - - "~>"
60
60
  - !ruby/object:Gem::Version
61
- version: '1.10'
61
+ version: '2.0'
62
62
  type: :development
63
63
  prerelease: false
64
64
  version_requirements: !ruby/object:Gem::Requirement
65
65
  requirements:
66
66
  - - "~>"
67
67
  - !ruby/object:Gem::Version
68
- version: '1.10'
68
+ version: '2.0'
69
69
  description: Concurrency and threshold throttling for Sidekiq.
70
70
  email:
71
71
  - ixti@member.fsf.org
@@ -74,6 +74,7 @@ extensions: []
74
74
  extra_rdoc_files: []
75
75
  files:
76
76
  - ".coveralls.yml"
77
+ - ".github/workflows/ci.yml"
77
78
  - ".gitignore"
78
79
  - ".rspec"
79
80
  - ".rubocop.yml"
@@ -90,10 +91,13 @@ files:
90
91
  - gemfiles/sidekiq_5.0.gemfile
91
92
  - gemfiles/sidekiq_5.1.gemfile
92
93
  - gemfiles/sidekiq_5.2.gemfile
94
+ - gemfiles/sidekiq_6.0.gemfile
95
+ - gemfiles/sidekiq_6.1.gemfile
93
96
  - lib/sidekiq/throttled.rb
94
97
  - lib/sidekiq/throttled/communicator.rb
95
98
  - lib/sidekiq/throttled/communicator/callbacks.rb
96
99
  - lib/sidekiq/throttled/communicator/listener.rb
100
+ - lib/sidekiq/throttled/configuration.rb
97
101
  - lib/sidekiq/throttled/errors.rb
98
102
  - lib/sidekiq/throttled/expirable_list.rb
99
103
  - lib/sidekiq/throttled/fetch.rb
@@ -110,6 +114,7 @@ files:
110
114
  - lib/sidekiq/throttled/strategy/threshold.lua
111
115
  - lib/sidekiq/throttled/strategy/threshold.rb
112
116
  - lib/sidekiq/throttled/testing.rb
117
+ - lib/sidekiq/throttled/utils.rb
113
118
  - lib/sidekiq/throttled/version.rb
114
119
  - lib/sidekiq/throttled/web.rb
115
120
  - lib/sidekiq/throttled/web/queues.html.erb
@@ -134,12 +139,11 @@ required_ruby_version: !ruby/object:Gem::Requirement
134
139
  version: '0'
135
140
  required_rubygems_version: !ruby/object:Gem::Requirement
136
141
  requirements:
137
- - - ">"
142
+ - - ">="
138
143
  - !ruby/object:Gem::Version
139
- version: 1.3.1
144
+ version: '0'
140
145
  requirements: []
141
- rubyforge_project:
142
- rubygems_version: 2.7.6
146
+ rubygems_version: 3.1.2
143
147
  signing_key:
144
148
  specification_version: 4
145
149
  summary: Concurrency and threshold throttling for Sidekiq.