sidekiq-throttled 0.14.1 → 0.16.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/dependabot.yml +12 -0
- data/.github/workflows/ci.yml +14 -16
- data/.rubocop.yml +8 -7
- data/.rubocop_todo.yml +39 -3
- data/Appraisals +14 -10
- data/CHANGES.md +77 -0
- data/Gemfile +7 -5
- data/LICENSE.md +2 -0
- data/README.md +9 -16
- data/Rakefile +1 -1
- data/gemfiles/sidekiq_6.0.gemfile +7 -5
- data/gemfiles/sidekiq_6.1.gemfile +7 -5
- data/gemfiles/{sidekiq_5.2.gemfile → sidekiq_6.2.gemfile} +8 -6
- data/gemfiles/{sidekiq_5.1.gemfile → sidekiq_6.3.gemfile} +8 -6
- data/gemfiles/{sidekiq_5.0.gemfile → sidekiq_6.4.gemfile} +8 -6
- data/gemfiles/sidekiq_6.5.gemfile +33 -0
- data/lib/sidekiq/throttled/communicator/callbacks.rb +5 -7
- data/lib/sidekiq/throttled/communicator/exception_handler.rb +25 -0
- data/lib/sidekiq/throttled/communicator/listener.rb +1 -1
- data/lib/sidekiq/throttled/communicator.rb +1 -1
- data/lib/sidekiq/throttled/expirable_list.rb +2 -5
- data/lib/sidekiq/throttled/fetch/unit_of_work.rb +7 -2
- data/lib/sidekiq/throttled/fetch.rb +5 -1
- data/lib/sidekiq/throttled/job.rb +128 -0
- data/lib/sidekiq/throttled/strategy/base.rb +6 -2
- data/lib/sidekiq/throttled/strategy_collection.rb +2 -3
- data/lib/sidekiq/throttled/version.rb +1 -1
- data/lib/sidekiq/throttled/web/stats.rb +5 -4
- data/lib/sidekiq/throttled/worker.rb +6 -121
- data/lib/sidekiq/throttled.rb +2 -1
- data/{.rubocop → rubocop}/layout.yml +0 -0
- data/{.rubocop → rubocop}/lint.yml +0 -0
- data/{.rubocop → rubocop}/metrics.yml +0 -0
- data/{.rubocop → rubocop}/performance.yml +0 -0
- data/{.rubocop → rubocop}/rspec.yml +1 -1
- data/{.rubocop → rubocop}/style.yml +0 -0
- data/sidekiq-throttled.gemspec +7 -5
- metadata +25 -20
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 51d312add58d0286d72087cefbdd49eac85a07c5416034e731f5c755f9fe702a
|
|
4
|
+
data.tar.gz: f7774ca65a0a26bb499dde8e34622d9e055b5112009228637a1f4b7696ad44bc
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 0e7170c5e6dbe4a5664523ab04794b2f921296c690fbac05e6a2966bc0285612374441ec86d561cf48a45d36369c8c2fe0154c99aa2b9ed347cec642a0c4d22a
|
|
7
|
+
data.tar.gz: a7c8ff62bb1ff8e09f609f8d341ab57539e3db8aa8112c29d09f052b487310c5969e98c5e80c5cc6f1b4e1dbd13a278f7d117bb4ff13e28ee0e2f75743044b74
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
|
|
2
|
+
|
|
3
|
+
version: 2
|
|
4
|
+
updates:
|
|
5
|
+
- package-ecosystem: "github-actions"
|
|
6
|
+
directory: "/"
|
|
7
|
+
schedule:
|
|
8
|
+
interval: "daily"
|
|
9
|
+
- package-ecosystem: "bundler"
|
|
10
|
+
directory: "/"
|
|
11
|
+
schedule:
|
|
12
|
+
interval: "daily"
|
data/.github/workflows/ci.yml
CHANGED
|
@@ -2,18 +2,19 @@ name: CI
|
|
|
2
2
|
|
|
3
3
|
on:
|
|
4
4
|
push:
|
|
5
|
-
branches: [
|
|
5
|
+
branches: [ main ]
|
|
6
6
|
pull_request:
|
|
7
|
-
branches: [
|
|
7
|
+
branches: [ main ]
|
|
8
8
|
|
|
9
9
|
jobs:
|
|
10
|
-
|
|
10
|
+
test:
|
|
11
11
|
name: "rspec (ruby:${{ matrix.ruby }} sidekiq:${{ matrix.sidekiq }})"
|
|
12
12
|
|
|
13
13
|
strategy:
|
|
14
|
+
fail-fast: false
|
|
14
15
|
matrix:
|
|
15
|
-
ruby: [ "2.
|
|
16
|
-
sidekiq: [ "
|
|
16
|
+
ruby: [ "2.7", "3.0", "3.1" ]
|
|
17
|
+
sidekiq: [ "6.0", "6.1", "6.2", "6.3", "6.4", "6.5" ]
|
|
17
18
|
|
|
18
19
|
runs-on: ubuntu-latest
|
|
19
20
|
|
|
@@ -24,31 +25,28 @@ jobs:
|
|
|
24
25
|
options: "--entrypoint redis-server"
|
|
25
26
|
|
|
26
27
|
env:
|
|
27
|
-
|
|
28
|
+
BUNDLE_GEMFILE: gemfiles/sidekiq_${{ matrix.sidekiq }}.gemfile
|
|
28
29
|
|
|
29
30
|
steps:
|
|
30
|
-
- uses: actions/checkout@
|
|
31
|
+
- uses: actions/checkout@v3
|
|
31
32
|
|
|
32
33
|
- uses: ruby/setup-ruby@v1
|
|
33
34
|
with:
|
|
34
35
|
ruby-version: ${{ matrix.ruby }}
|
|
36
|
+
bundler-cache: true
|
|
35
37
|
|
|
36
|
-
- name: bundle
|
|
37
|
-
run: bundle
|
|
38
|
-
|
|
39
|
-
- run: bundle exec rspec
|
|
38
|
+
- name: bundle exec rspec
|
|
39
|
+
run: bundle exec rspec --format progress --force-colour
|
|
40
40
|
|
|
41
41
|
rubocop:
|
|
42
42
|
runs-on: ubuntu-latest
|
|
43
43
|
|
|
44
44
|
steps:
|
|
45
|
-
- uses: actions/checkout@
|
|
45
|
+
- uses: actions/checkout@v3
|
|
46
46
|
|
|
47
47
|
- uses: ruby/setup-ruby@v1
|
|
48
48
|
with:
|
|
49
|
-
ruby-version: "2.
|
|
50
|
-
|
|
51
|
-
- name: bundle install
|
|
52
|
-
run: bundle install --without development --jobs 4 --retry 3
|
|
49
|
+
ruby-version: "2.7"
|
|
50
|
+
bundler-cache: true
|
|
53
51
|
|
|
54
52
|
- run: bundle exec rubocop
|
data/.rubocop.yml
CHANGED
|
@@ -1,19 +1,20 @@
|
|
|
1
1
|
require:
|
|
2
2
|
- rubocop-performance
|
|
3
|
+
- rubocop-rake
|
|
3
4
|
- rubocop-rspec
|
|
4
5
|
|
|
5
6
|
inherit_from:
|
|
6
7
|
- .rubocop_todo.yml
|
|
7
|
-
-
|
|
8
|
-
-
|
|
9
|
-
-
|
|
10
|
-
-
|
|
11
|
-
-
|
|
12
|
-
-
|
|
8
|
+
- rubocop/layout.yml
|
|
9
|
+
- rubocop/lint.yml
|
|
10
|
+
- rubocop/metrics.yml
|
|
11
|
+
- rubocop/performance.yml
|
|
12
|
+
- rubocop/rspec.yml
|
|
13
|
+
- rubocop/style.yml
|
|
13
14
|
|
|
14
15
|
AllCops:
|
|
15
16
|
Exclude:
|
|
16
17
|
- gemfiles/**/*
|
|
17
18
|
- vendor/**/*
|
|
18
19
|
NewCops: enable
|
|
19
|
-
TargetRubyVersion: 2.
|
|
20
|
+
TargetRubyVersion: 2.7
|
data/.rubocop_todo.yml
CHANGED
|
@@ -1,12 +1,40 @@
|
|
|
1
1
|
# This configuration was generated by
|
|
2
2
|
# `rubocop --auto-gen-config`
|
|
3
|
-
# on
|
|
3
|
+
# on 2022-06-13 00:11:28 UTC using RuboCop version 1.30.1.
|
|
4
4
|
# The point is for the user to remove these configuration records
|
|
5
5
|
# one by one as the offenses are removed from the code base.
|
|
6
6
|
# Note that changes in the inspected code, or installation of new
|
|
7
7
|
# versions of RuboCop, may require this file to be generated again.
|
|
8
8
|
|
|
9
|
-
# Offense count:
|
|
9
|
+
# Offense count: 3
|
|
10
|
+
# Configuration parameters: AllowComments, AllowEmptyLambdas.
|
|
11
|
+
Lint/EmptyBlock:
|
|
12
|
+
Exclude:
|
|
13
|
+
- 'spec/sidekiq/throttled/middleware_spec.rb'
|
|
14
|
+
- 'spec/sidekiq/throttled/registry_spec.rb'
|
|
15
|
+
|
|
16
|
+
# Offense count: 2
|
|
17
|
+
# Configuration parameters: EnforcedStyleForLeadingUnderscores.
|
|
18
|
+
# SupportedStylesForLeadingUnderscores: disallowed, required, optional
|
|
19
|
+
Naming/MemoizedInstanceVariableName:
|
|
20
|
+
Exclude:
|
|
21
|
+
- 'lib/sidekiq/throttled/communicator.rb'
|
|
22
|
+
- 'lib/sidekiq/throttled/queues_pauser.rb'
|
|
23
|
+
|
|
24
|
+
# Offense count: 2
|
|
25
|
+
Performance/MethodObjectAsBlock:
|
|
26
|
+
Exclude:
|
|
27
|
+
- 'lib/sidekiq/throttled/queues_pauser.rb'
|
|
28
|
+
|
|
29
|
+
# Offense count: 2
|
|
30
|
+
# Configuration parameters: Include, CustomTransform, IgnoreMethods, SpecSuffixOnly.
|
|
31
|
+
# Include: **/*_spec*rb*, **/spec/**/*
|
|
32
|
+
RSpec/FilePath:
|
|
33
|
+
Exclude:
|
|
34
|
+
- 'spec/sidekiq/throttled/web/queues_spec.rb'
|
|
35
|
+
- 'spec/sidekiq/throttled/web/throttled_spec.rb'
|
|
36
|
+
|
|
37
|
+
# Offense count: 69
|
|
10
38
|
# Configuration parameters: .
|
|
11
39
|
# SupportedStyles: have_received, receive
|
|
12
40
|
RSpec/MessageSpies:
|
|
@@ -25,8 +53,16 @@ RSpec/MultipleMemoizedHelpers:
|
|
|
25
53
|
RSpec/NestedGroups:
|
|
26
54
|
Max: 5
|
|
27
55
|
|
|
28
|
-
# Offense count:
|
|
56
|
+
# Offense count: 5
|
|
57
|
+
RSpec/StubbedMock:
|
|
58
|
+
Exclude:
|
|
59
|
+
- 'spec/sidekiq/throttled/expirable_list_spec.rb'
|
|
60
|
+
- 'spec/sidekiq/throttled/fetch_spec.rb'
|
|
61
|
+
- 'spec/sidekiq/throttled/queues_pauser_spec.rb'
|
|
62
|
+
|
|
63
|
+
# Offense count: 6
|
|
29
64
|
RSpec/SubjectStub:
|
|
30
65
|
Exclude:
|
|
31
66
|
- 'spec/sidekiq/throttled/communicator_spec.rb'
|
|
67
|
+
- 'spec/sidekiq/throttled/fetch_spec.rb'
|
|
32
68
|
- 'spec/sidekiq/throttled/queues_pauser_spec.rb'
|
data/Appraisals
CHANGED
|
@@ -1,21 +1,25 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
appraise "sidekiq-
|
|
4
|
-
gem "sidekiq", "~>
|
|
3
|
+
appraise "sidekiq-6.0" do
|
|
4
|
+
gem "sidekiq", "~> 6.0.0"
|
|
5
5
|
end
|
|
6
6
|
|
|
7
|
-
appraise "sidekiq-
|
|
8
|
-
gem "sidekiq", "~>
|
|
7
|
+
appraise "sidekiq-6.1" do
|
|
8
|
+
gem "sidekiq", "~> 6.1.0"
|
|
9
9
|
end
|
|
10
10
|
|
|
11
|
-
appraise "sidekiq-
|
|
12
|
-
gem "sidekiq", "~>
|
|
11
|
+
appraise "sidekiq-6.2" do
|
|
12
|
+
gem "sidekiq", "~> 6.2.0"
|
|
13
13
|
end
|
|
14
14
|
|
|
15
|
-
appraise "sidekiq-6.
|
|
16
|
-
gem "sidekiq", "~> 6.
|
|
15
|
+
appraise "sidekiq-6.3" do
|
|
16
|
+
gem "sidekiq", "~> 6.3.0"
|
|
17
17
|
end
|
|
18
18
|
|
|
19
|
-
appraise "sidekiq-6.
|
|
20
|
-
gem "sidekiq", "~> 6.
|
|
19
|
+
appraise "sidekiq-6.4" do
|
|
20
|
+
gem "sidekiq", "~> 6.4.0"
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
appraise "sidekiq-6.5" do
|
|
24
|
+
gem "sidekiq", "~> 6.5.0"
|
|
21
25
|
end
|
data/CHANGES.md
CHANGED
|
@@ -1,3 +1,66 @@
|
|
|
1
|
+
## 0.16.1 (2022-06-14)
|
|
2
|
+
|
|
3
|
+
* [#2](https://github.com/ixti/sidekiq-throttled/pull/2)
|
|
4
|
+
Support custom ActiveJob adapters.
|
|
5
|
+
([@longkt90])
|
|
6
|
+
|
|
7
|
+
* [#107](https://github.com/sensortower/sidekiq-throttled/pull/107)
|
|
8
|
+
Log error when key sufix extraction fails.
|
|
9
|
+
([@pjungwir])
|
|
10
|
+
|
|
11
|
+
## 0.16.0 (2022-06-13)
|
|
12
|
+
|
|
13
|
+
* Drop Ruby 2.6 support.
|
|
14
|
+
|
|
15
|
+
* Drop Sidekiq 5.X support.
|
|
16
|
+
|
|
17
|
+
* [#121](https://github.com/sensortower/sidekiq-throttled/pull/121)
|
|
18
|
+
Sidekiq 6.5 compatibility.
|
|
19
|
+
([@hieuk09])
|
|
20
|
+
|
|
21
|
+
* [#116](https://github.com/sensortower/sidekiq-throttled/pull/116)
|
|
22
|
+
Unwrap ActiveJob JobWrapper
|
|
23
|
+
([@holstvoogd])
|
|
24
|
+
|
|
25
|
+
* [#115](https://github.com/sensortower/sidekiq-throttled/pull/115)
|
|
26
|
+
Fix Redis 4.6 deprecation warnings.
|
|
27
|
+
([@dbackeus])
|
|
28
|
+
|
|
29
|
+
* [#113](https://github.com/sensortower/sidekiq-throttled/pull/113)
|
|
30
|
+
Add Ruby 3.1 to CI.
|
|
31
|
+
([@petergoldstein])
|
|
32
|
+
|
|
33
|
+
* [#103](https://github.com/sensortower/sidekiq-throttled/pull/103)
|
|
34
|
+
Rename Sidekiq::Throttled::Worker to Sidekiq::Throttled::Job, and alias it
|
|
35
|
+
as Sidekiq::Throttled::Worker.
|
|
36
|
+
([@CHTJonas])
|
|
37
|
+
|
|
38
|
+
## 0.15.0 (2021-12-16)
|
|
39
|
+
|
|
40
|
+
* [#102](https://github.com/sensortower/sidekiq-throttled/pull/102)
|
|
41
|
+
Support Ruby 3.0 and Sidekiq 6.2+.
|
|
42
|
+
([@ybiquitous])
|
|
43
|
+
|
|
44
|
+
* [#97](https://github.com/sensortower/sidekiq-throttled/pull/97)
|
|
45
|
+
Fix kwargs usage in strategy collection.
|
|
46
|
+
([@baptistejub])
|
|
47
|
+
|
|
48
|
+
## 0.14.0 (2021-09-21)
|
|
49
|
+
|
|
50
|
+
* [#98](https://github.com/sensortower/sidekiq-throttled/pull/98)
|
|
51
|
+
Remove warning for strategy override.
|
|
52
|
+
([@mattiagiuffrida-st])
|
|
53
|
+
|
|
54
|
+
* [#65](https://github.com/sensortower/sidekiq-throttled/pull/65)
|
|
55
|
+
Support composite (multi-key) strategies.
|
|
56
|
+
([@holyketzer])
|
|
57
|
+
|
|
58
|
+
* [#89](https://github.com/sensortower/sidekiq-throttled/pull/89),
|
|
59
|
+
[#96](https://github.com/sensortower/sidekiq-throttled/pull/96), and
|
|
60
|
+
[#93](https://github.com/sensortower/sidekiq-throttled/pull/93)
|
|
61
|
+
Improve documentation.
|
|
62
|
+
([@hubertjakubiak], [@khaile], and [@kylerippey])
|
|
63
|
+
|
|
1
64
|
## 0.13.0 (2020-07-28)
|
|
2
65
|
|
|
3
66
|
* [#85](https://github.com/sensortower/sidekiq-throttled/pull/85)
|
|
@@ -221,3 +284,17 @@
|
|
|
221
284
|
[@lenon]: https://github.com/lenon
|
|
222
285
|
[@vaot]: https://github.com/vaot
|
|
223
286
|
[@hmaack]: https://github.com/hmaack
|
|
287
|
+
[@holyketzer]: https://github.com/holyketzer
|
|
288
|
+
[@hubertjakubiak]: https://github.com/hubertjakubiak
|
|
289
|
+
[@kylerippey]: https://github.com/kylerippey
|
|
290
|
+
[@khaile]: https://github.com/khaile
|
|
291
|
+
[@mattiagiuffrida-st]: https://github.com/mattiagiuffrida-st
|
|
292
|
+
[@baptistejub]: https://github.com/baptistejub
|
|
293
|
+
[@ybiquitous]: https://github.com/ybiquitous
|
|
294
|
+
[@hieuk09]: https://github.com/hieuk09
|
|
295
|
+
[@petergoldstein]: https://github.com/petergoldstein
|
|
296
|
+
[@dbackeus]: https://github.com/dbackeus
|
|
297
|
+
[@holstvoogd]: https://github.com/holstvoogd
|
|
298
|
+
[@CHTJonas]: https://github.com/CHTJonas
|
|
299
|
+
[@pjungwir]: https://github.com/pjungwir
|
|
300
|
+
[@longkt90]: https://github.com/longkt90
|
data/Gemfile
CHANGED
|
@@ -5,9 +5,6 @@ source "https://rubygems.org"
|
|
|
5
5
|
gem "appraisal"
|
|
6
6
|
gem "rake"
|
|
7
7
|
gem "rspec"
|
|
8
|
-
gem "rubocop", "~> 0.90.0", :require => false
|
|
9
|
-
gem "rubocop-performance", "~> 1.8.0", :require => false
|
|
10
|
-
gem "rubocop-rspec", "~> 1.43.2", :require => false
|
|
11
8
|
gem "sidekiq"
|
|
12
9
|
|
|
13
10
|
group :development do
|
|
@@ -20,13 +17,18 @@ end
|
|
|
20
17
|
group :test do
|
|
21
18
|
gem "apparition"
|
|
22
19
|
gem "capybara"
|
|
23
|
-
gem "coveralls", :require => false
|
|
24
20
|
gem "puma"
|
|
25
21
|
gem "rack-test"
|
|
26
|
-
gem "simplecov"
|
|
27
22
|
gem "sinatra"
|
|
28
23
|
gem "timecop"
|
|
29
24
|
end
|
|
30
25
|
|
|
26
|
+
group :lint do
|
|
27
|
+
gem "rubocop", :require => false
|
|
28
|
+
gem "rubocop-performance", :require => false
|
|
29
|
+
gem "rubocop-rake", :require => false
|
|
30
|
+
gem "rubocop-rspec", :require => false
|
|
31
|
+
end
|
|
32
|
+
|
|
31
33
|
# Specify your gem's dependencies in sidekiq-throttled.gemspec
|
|
32
34
|
gemspec
|
data/LICENSE.md
CHANGED
data/README.md
CHANGED
|
@@ -1,10 +1,7 @@
|
|
|
1
1
|
# Sidekiq::Throttled
|
|
2
2
|
|
|
3
|
+
[](https://github.com/ixti/sidekiq-throttled/actions/workflows/ci.yml)
|
|
3
4
|
[](http://rubygems.org/gems/sidekiq-throttled)
|
|
4
|
-
[](https://github.com/sensortower/sidekiq-throttled/actions?query=workflow%3ACI+branch%3Amaster)
|
|
5
|
-
[](https://codeclimate.com/github/sensortower/sidekiq-throttled)
|
|
6
|
-
[](https://coveralls.io/github/sensortower/sidekiq-throttled?branch=master)
|
|
7
|
-
[](http://inch-ci.org/github/sensortower/sidekiq-throttled)
|
|
8
5
|
[](http://www.rubydoc.info/gems/sidekiq-throttled)
|
|
9
6
|
|
|
10
7
|
Concurrency and threshold throttling for [Sidekiq][sidekiq].
|
|
@@ -230,11 +227,12 @@ end
|
|
|
230
227
|
|
|
231
228
|
## Supported Ruby Versions
|
|
232
229
|
|
|
233
|
-
This library aims to support and is [tested against][
|
|
230
|
+
This library aims to support and is [tested against][ci] the following Ruby
|
|
234
231
|
versions:
|
|
235
232
|
|
|
236
|
-
* Ruby 2.6.x
|
|
237
233
|
* Ruby 2.7.x
|
|
234
|
+
* Ruby 3.0.x
|
|
235
|
+
* Ruby 3.1.x
|
|
238
236
|
|
|
239
237
|
If something doesn't work on one of these versions, it's a bug.
|
|
240
238
|
|
|
@@ -254,11 +252,12 @@ dropped.
|
|
|
254
252
|
|
|
255
253
|
This library aims to support work with following [Sidekiq][sidekiq] versions:
|
|
256
254
|
|
|
257
|
-
* Sidekiq 5.0.x
|
|
258
|
-
* Sidekiq 5.1.x
|
|
259
|
-
* Sidekiq 5.2.x
|
|
260
255
|
* Sidekiq 6.0.x
|
|
261
256
|
* Sidekiq 6.1.x
|
|
257
|
+
* Sidekiq 6.2.x
|
|
258
|
+
* Sidekiq 6.3.x
|
|
259
|
+
* Sidekiq 6.4.x
|
|
260
|
+
* Sidekiq 6.5.x
|
|
262
261
|
|
|
263
262
|
|
|
264
263
|
## Contributing
|
|
@@ -284,11 +283,5 @@ bundle exec rubocop # run static code analysis
|
|
|
284
283
|
Don't forget to run `appraisal update` after any changes to `Gemfile`.
|
|
285
284
|
|
|
286
285
|
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
Copyright (c) 2015-2020 SensorTower Inc.
|
|
290
|
-
See LICENSE.md for further details.
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
[travis]: http://travis-ci.org/sensortower/sidekiq-throttled
|
|
286
|
+
[ci]: https://github.com/ixti/sidekiq-throttled/actions/workflows/ci.yml
|
|
294
287
|
[sidekiq]: https://github.com/mperham/sidekiq
|
data/Rakefile
CHANGED
|
@@ -24,4 +24,4 @@ end
|
|
|
24
24
|
default_suite = ENV["CI"] ? :spec : %i[spec rubocop]
|
|
25
25
|
named_suites = { "rubocop" => :rubocop, "rspec" => :spec }
|
|
26
26
|
|
|
27
|
-
task :default => named_suites.fetch(ENV
|
|
27
|
+
task :default => named_suites.fetch(ENV.fetch("SUITE", nil), default_suite)
|
|
@@ -5,9 +5,6 @@ source "https://rubygems.org"
|
|
|
5
5
|
gem "appraisal"
|
|
6
6
|
gem "rake"
|
|
7
7
|
gem "rspec"
|
|
8
|
-
gem "rubocop", "~> 0.82.0", require: false
|
|
9
|
-
gem "rubocop-performance", "~>1.5.2", require: false
|
|
10
|
-
gem "rubocop-rspec", "~> 1.39.0", require: false
|
|
11
8
|
gem "sidekiq", "~> 6.0.0"
|
|
12
9
|
|
|
13
10
|
group :development do
|
|
@@ -20,12 +17,17 @@ end
|
|
|
20
17
|
group :test do
|
|
21
18
|
gem "apparition"
|
|
22
19
|
gem "capybara"
|
|
23
|
-
gem "coveralls", require: false
|
|
24
20
|
gem "puma"
|
|
25
21
|
gem "rack-test"
|
|
26
|
-
gem "simplecov"
|
|
27
22
|
gem "sinatra"
|
|
28
23
|
gem "timecop"
|
|
29
24
|
end
|
|
30
25
|
|
|
26
|
+
group :lint do
|
|
27
|
+
gem "rubocop", require: false
|
|
28
|
+
gem "rubocop-performance", require: false
|
|
29
|
+
gem "rubocop-rake", require: false
|
|
30
|
+
gem "rubocop-rspec", require: false
|
|
31
|
+
end
|
|
32
|
+
|
|
31
33
|
gemspec path: "../"
|
|
@@ -5,9 +5,6 @@ source "https://rubygems.org"
|
|
|
5
5
|
gem "appraisal"
|
|
6
6
|
gem "rake"
|
|
7
7
|
gem "rspec"
|
|
8
|
-
gem "rubocop", "~> 0.82.0", require: false
|
|
9
|
-
gem "rubocop-performance", "~>1.5.2", require: false
|
|
10
|
-
gem "rubocop-rspec", "~> 1.39.0", require: false
|
|
11
8
|
gem "sidekiq", "~> 6.1.0"
|
|
12
9
|
|
|
13
10
|
group :development do
|
|
@@ -20,12 +17,17 @@ end
|
|
|
20
17
|
group :test do
|
|
21
18
|
gem "apparition"
|
|
22
19
|
gem "capybara"
|
|
23
|
-
gem "coveralls", require: false
|
|
24
20
|
gem "puma"
|
|
25
21
|
gem "rack-test"
|
|
26
|
-
gem "simplecov"
|
|
27
22
|
gem "sinatra"
|
|
28
23
|
gem "timecop"
|
|
29
24
|
end
|
|
30
25
|
|
|
26
|
+
group :lint do
|
|
27
|
+
gem "rubocop", require: false
|
|
28
|
+
gem "rubocop-performance", require: false
|
|
29
|
+
gem "rubocop-rake", require: false
|
|
30
|
+
gem "rubocop-rspec", require: false
|
|
31
|
+
end
|
|
32
|
+
|
|
31
33
|
gemspec path: "../"
|
|
@@ -5,10 +5,7 @@ source "https://rubygems.org"
|
|
|
5
5
|
gem "appraisal"
|
|
6
6
|
gem "rake"
|
|
7
7
|
gem "rspec"
|
|
8
|
-
gem "
|
|
9
|
-
gem "rubocop-performance", "~>1.5.2", require: false
|
|
10
|
-
gem "rubocop-rspec", "~> 1.39.0", require: false
|
|
11
|
-
gem "sidekiq", "~> 5.2.0"
|
|
8
|
+
gem "sidekiq", "~> 6.2.0"
|
|
12
9
|
|
|
13
10
|
group :development do
|
|
14
11
|
gem "byebug"
|
|
@@ -20,12 +17,17 @@ end
|
|
|
20
17
|
group :test do
|
|
21
18
|
gem "apparition"
|
|
22
19
|
gem "capybara"
|
|
23
|
-
gem "coveralls", require: false
|
|
24
20
|
gem "puma"
|
|
25
21
|
gem "rack-test"
|
|
26
|
-
gem "simplecov"
|
|
27
22
|
gem "sinatra"
|
|
28
23
|
gem "timecop"
|
|
29
24
|
end
|
|
30
25
|
|
|
26
|
+
group :lint do
|
|
27
|
+
gem "rubocop", require: false
|
|
28
|
+
gem "rubocop-performance", require: false
|
|
29
|
+
gem "rubocop-rake", require: false
|
|
30
|
+
gem "rubocop-rspec", require: false
|
|
31
|
+
end
|
|
32
|
+
|
|
31
33
|
gemspec path: "../"
|
|
@@ -5,10 +5,7 @@ source "https://rubygems.org"
|
|
|
5
5
|
gem "appraisal"
|
|
6
6
|
gem "rake"
|
|
7
7
|
gem "rspec"
|
|
8
|
-
gem "
|
|
9
|
-
gem "rubocop-performance", "~>1.5.2", require: false
|
|
10
|
-
gem "rubocop-rspec", "~> 1.39.0", require: false
|
|
11
|
-
gem "sidekiq", "~> 5.1.0"
|
|
8
|
+
gem "sidekiq", "~> 6.3.0"
|
|
12
9
|
|
|
13
10
|
group :development do
|
|
14
11
|
gem "byebug"
|
|
@@ -20,12 +17,17 @@ end
|
|
|
20
17
|
group :test do
|
|
21
18
|
gem "apparition"
|
|
22
19
|
gem "capybara"
|
|
23
|
-
gem "coveralls", require: false
|
|
24
20
|
gem "puma"
|
|
25
21
|
gem "rack-test"
|
|
26
|
-
gem "simplecov"
|
|
27
22
|
gem "sinatra"
|
|
28
23
|
gem "timecop"
|
|
29
24
|
end
|
|
30
25
|
|
|
26
|
+
group :lint do
|
|
27
|
+
gem "rubocop", require: false
|
|
28
|
+
gem "rubocop-performance", require: false
|
|
29
|
+
gem "rubocop-rake", require: false
|
|
30
|
+
gem "rubocop-rspec", require: false
|
|
31
|
+
end
|
|
32
|
+
|
|
31
33
|
gemspec path: "../"
|
|
@@ -5,10 +5,7 @@ source "https://rubygems.org"
|
|
|
5
5
|
gem "appraisal"
|
|
6
6
|
gem "rake"
|
|
7
7
|
gem "rspec"
|
|
8
|
-
gem "
|
|
9
|
-
gem "rubocop-performance", "~>1.5.2", require: false
|
|
10
|
-
gem "rubocop-rspec", "~> 1.39.0", require: false
|
|
11
|
-
gem "sidekiq", "~> 5.0.0"
|
|
8
|
+
gem "sidekiq", "~> 6.4.0"
|
|
12
9
|
|
|
13
10
|
group :development do
|
|
14
11
|
gem "byebug"
|
|
@@ -20,12 +17,17 @@ end
|
|
|
20
17
|
group :test do
|
|
21
18
|
gem "apparition"
|
|
22
19
|
gem "capybara"
|
|
23
|
-
gem "coveralls", require: false
|
|
24
20
|
gem "puma"
|
|
25
21
|
gem "rack-test"
|
|
26
|
-
gem "simplecov"
|
|
27
22
|
gem "sinatra"
|
|
28
23
|
gem "timecop"
|
|
29
24
|
end
|
|
30
25
|
|
|
26
|
+
group :lint do
|
|
27
|
+
gem "rubocop", require: false
|
|
28
|
+
gem "rubocop-performance", require: false
|
|
29
|
+
gem "rubocop-rake", require: false
|
|
30
|
+
gem "rubocop-rspec", require: false
|
|
31
|
+
end
|
|
32
|
+
|
|
31
33
|
gemspec path: "../"
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
# This file was generated by Appraisal
|
|
2
|
+
|
|
3
|
+
source "https://rubygems.org"
|
|
4
|
+
|
|
5
|
+
gem "appraisal"
|
|
6
|
+
gem "rake"
|
|
7
|
+
gem "rspec"
|
|
8
|
+
gem "sidekiq", "~> 6.5.0"
|
|
9
|
+
|
|
10
|
+
group :development do
|
|
11
|
+
gem "byebug"
|
|
12
|
+
gem "guard", require: false
|
|
13
|
+
gem "guard-rspec", require: false
|
|
14
|
+
gem "guard-rubocop", require: false
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
group :test do
|
|
18
|
+
gem "apparition"
|
|
19
|
+
gem "capybara"
|
|
20
|
+
gem "puma"
|
|
21
|
+
gem "rack-test"
|
|
22
|
+
gem "sinatra"
|
|
23
|
+
gem "timecop"
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
group :lint do
|
|
27
|
+
gem "rubocop", require: false
|
|
28
|
+
gem "rubocop-performance", require: false
|
|
29
|
+
gem "rubocop-rake", require: false
|
|
30
|
+
gem "rubocop-rspec", require: false
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
gemspec path: "../"
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
require "fiber"
|
|
4
4
|
|
|
5
|
-
require "sidekiq/exception_handler"
|
|
5
|
+
require "sidekiq/throttled/communicator/exception_handler"
|
|
6
6
|
|
|
7
7
|
module Sidekiq
|
|
8
8
|
module Throttled
|
|
@@ -53,15 +53,13 @@ module Sidekiq
|
|
|
53
53
|
# @param [#to_s] event
|
|
54
54
|
# @param [Object] payload
|
|
55
55
|
# @return [void]
|
|
56
|
-
def run(event, payload = nil)
|
|
56
|
+
def run(event, payload = nil)
|
|
57
57
|
@mutex.synchronize do
|
|
58
58
|
fiber = Fiber.new do
|
|
59
59
|
@handlers[event.to_s].each do |callback|
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
handle_exception(e, :context => "sidekiq:throttled")
|
|
64
|
-
end
|
|
60
|
+
callback.call(payload)
|
|
61
|
+
rescue => e
|
|
62
|
+
handle_exception(e, :context => "sidekiq:throttled")
|
|
65
63
|
end
|
|
66
64
|
end
|
|
67
65
|
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
require "sidekiq"
|
|
4
|
+
require "sidekiq/version"
|
|
5
|
+
|
|
6
|
+
module Sidekiq
|
|
7
|
+
module Throttled
|
|
8
|
+
class Communicator
|
|
9
|
+
if Sidekiq::VERSION >= "6.5.0"
|
|
10
|
+
module ExceptionHandler
|
|
11
|
+
def handle_exception(*args)
|
|
12
|
+
Sidekiq.handle_exception(*args)
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# NOTE: `Sidekiq.default_error_handler` is private API
|
|
17
|
+
Sidekiq.error_handlers << Sidekiq.method(:default_error_handler)
|
|
18
|
+
else
|
|
19
|
+
require "sidekiq/exception_handler"
|
|
20
|
+
|
|
21
|
+
ExceptionHandler = ::Sidekiq::ExceptionHandler
|
|
22
|
+
end
|
|
23
|
+
end
|
|
24
|
+
end
|
|
25
|
+
end
|
|
@@ -2,8 +2,6 @@
|
|
|
2
2
|
|
|
3
3
|
require "monitor"
|
|
4
4
|
|
|
5
|
-
require "concurrent/utility/monotonic_time"
|
|
6
|
-
|
|
7
5
|
module Sidekiq
|
|
8
6
|
module Throttled
|
|
9
7
|
# List that tracks when elements were added and enumerates over those not
|
|
@@ -24,7 +22,6 @@ module Sidekiq
|
|
|
24
22
|
# It does not deduplicates elements. Eviction happens only upon elements
|
|
25
23
|
# retrieval (see {#each}).
|
|
26
24
|
#
|
|
27
|
-
# @see http://ruby-concurrency.github.io/concurrent-ruby/Concurrent.html#monotonic_time-class_method
|
|
28
25
|
# @see https://ruby-doc.org/core/Process.html#method-c-clock_gettime
|
|
29
26
|
# @see https://linux.die.net/man/3/clock_gettime
|
|
30
27
|
#
|
|
@@ -44,7 +41,7 @@ module Sidekiq
|
|
|
44
41
|
# @params element [Object]
|
|
45
42
|
# @return [ExpirableList] self
|
|
46
43
|
def <<(element)
|
|
47
|
-
@mon.synchronize { @arr << [
|
|
44
|
+
@mon.synchronize { @arr << [::Process.clock_gettime(::Process::CLOCK_MONOTONIC), element] }
|
|
48
45
|
self
|
|
49
46
|
end
|
|
50
47
|
|
|
@@ -58,7 +55,7 @@ module Sidekiq
|
|
|
58
55
|
return to_enum __method__ unless block_given?
|
|
59
56
|
|
|
60
57
|
@mon.synchronize do
|
|
61
|
-
horizon =
|
|
58
|
+
horizon = ::Process.clock_gettime(::Process::CLOCK_MONOTONIC) - @ttl
|
|
62
59
|
|
|
63
60
|
# drop all elements older than horizon
|
|
64
61
|
@arr.shift while @arr[0] && @arr[0][0] < horizon
|
|
@@ -49,9 +49,14 @@ module Sidekiq
|
|
|
49
49
|
# process was terminated. It is a reverse of whatever fetcher was
|
|
50
50
|
# doing to pull the job out of queue.
|
|
51
51
|
#
|
|
52
|
+
# @param [Redis] pipelined connection for requeing via Redis#pipelined
|
|
52
53
|
# @return [void]
|
|
53
|
-
def requeue
|
|
54
|
-
|
|
54
|
+
def requeue(pipeline = nil)
|
|
55
|
+
if pipeline
|
|
56
|
+
pipeline.rpush(QueueName.expand(queue_name), job)
|
|
57
|
+
else
|
|
58
|
+
Sidekiq.redis { |conn| conn.rpush(QueueName.expand(queue_name), job) }
|
|
59
|
+
end
|
|
55
60
|
end
|
|
56
61
|
|
|
57
62
|
# Pushes job back to the head of the queue, so that job won't be tried
|
|
@@ -22,7 +22,11 @@ module Sidekiq
|
|
|
22
22
|
return if units.empty?
|
|
23
23
|
|
|
24
24
|
Sidekiq.logger.debug { "Re-queueing terminated jobs" }
|
|
25
|
-
Sidekiq.redis
|
|
25
|
+
Sidekiq.redis do |conn|
|
|
26
|
+
conn.pipelined do |pipeline|
|
|
27
|
+
units.each { |unit| unit.requeue(pipeline) }
|
|
28
|
+
end
|
|
29
|
+
end
|
|
26
30
|
Sidekiq.logger.info("Pushed #{units.size} jobs back to Redis")
|
|
27
31
|
rescue => e
|
|
28
32
|
Sidekiq.logger.warn("Failed to requeue #{units.size} jobs: #{e}")
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
# internal
|
|
4
|
+
require "sidekiq/throttled/registry"
|
|
5
|
+
|
|
6
|
+
module Sidekiq
|
|
7
|
+
module Throttled
|
|
8
|
+
# Adds helpers to your worker classes
|
|
9
|
+
#
|
|
10
|
+
# @example Usage
|
|
11
|
+
#
|
|
12
|
+
# class MyWorker
|
|
13
|
+
# include Sidekiq::Worker
|
|
14
|
+
# include Sidekiq::Throttled::Worker
|
|
15
|
+
#
|
|
16
|
+
# sidkiq_options :queue => :my_queue
|
|
17
|
+
# sidekiq_throttle :threshold => { :limit => 123, :period => 1.hour }
|
|
18
|
+
#
|
|
19
|
+
# def perform
|
|
20
|
+
# # ...
|
|
21
|
+
# end
|
|
22
|
+
# end
|
|
23
|
+
#
|
|
24
|
+
# @see ClassMethods
|
|
25
|
+
module Job
|
|
26
|
+
# Extends worker class with {ClassMethods}.
|
|
27
|
+
#
|
|
28
|
+
# @note Using `included` hook with extending worker with {ClassMethods}
|
|
29
|
+
# in order to make API inline with `include Sidekiq::Worker`.
|
|
30
|
+
#
|
|
31
|
+
# @private
|
|
32
|
+
def self.included(worker)
|
|
33
|
+
worker.send(:extend, ClassMethods)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Helper methods added to the singleton class of destination
|
|
37
|
+
module ClassMethods
|
|
38
|
+
# Registers some strategy for the worker.
|
|
39
|
+
#
|
|
40
|
+
# @example Allow max 123 MyWorker jobs per hour
|
|
41
|
+
#
|
|
42
|
+
# class MyWorker
|
|
43
|
+
# include Sidekiq::Worker
|
|
44
|
+
# include Sidekiq::Throttled::Worker
|
|
45
|
+
#
|
|
46
|
+
# sidekiq_throttle({
|
|
47
|
+
# :threshold => { :limit => 123, :period => 1.hour }
|
|
48
|
+
# })
|
|
49
|
+
# end
|
|
50
|
+
#
|
|
51
|
+
# @example Allow max 10 concurrently running MyWorker jobs
|
|
52
|
+
#
|
|
53
|
+
# class MyWorker
|
|
54
|
+
# include Sidekiq::Worker
|
|
55
|
+
# include Sidekiq::Throttled::Worker
|
|
56
|
+
#
|
|
57
|
+
# sidekiq_throttle({
|
|
58
|
+
# :concurrency => { :limit => 10 }
|
|
59
|
+
# })
|
|
60
|
+
# end
|
|
61
|
+
#
|
|
62
|
+
# @example Allow max 10 concurrent MyWorker jobs and max 123 per hour
|
|
63
|
+
#
|
|
64
|
+
# class MyWorker
|
|
65
|
+
# include Sidekiq::Worker
|
|
66
|
+
# include Sidekiq::Throttled::Worker
|
|
67
|
+
#
|
|
68
|
+
# sidekiq_throttle({
|
|
69
|
+
# :threshold => { :limit => 123, :period => 1.hour },
|
|
70
|
+
# :concurrency => { :limit => 10 }
|
|
71
|
+
# })
|
|
72
|
+
# end
|
|
73
|
+
#
|
|
74
|
+
# @see Registry.add
|
|
75
|
+
# @return [void]
|
|
76
|
+
def sidekiq_throttle(**kwargs)
|
|
77
|
+
Registry.add(self, **kwargs)
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
# Adds current worker to preconfigured throttling strategy. Allows
|
|
81
|
+
# sharing same pool for multiple workers.
|
|
82
|
+
#
|
|
83
|
+
# First of all we need to create shared throttling strategy:
|
|
84
|
+
#
|
|
85
|
+
# # Create google_api throttling strategy
|
|
86
|
+
# Sidekiq::Throttled::Registry.add(:google_api, {
|
|
87
|
+
# :threshold => { :limit => 123, :period => 1.hour },
|
|
88
|
+
# :concurrency => { :limit => 10 }
|
|
89
|
+
# })
|
|
90
|
+
#
|
|
91
|
+
# Now we can assign it to our workers:
|
|
92
|
+
#
|
|
93
|
+
# class FetchProfileJob
|
|
94
|
+
# include Sidekiq::Worker
|
|
95
|
+
# include Sidekiq::Throttled::Worker
|
|
96
|
+
#
|
|
97
|
+
# sidekiq_throttle_as :google_api
|
|
98
|
+
# end
|
|
99
|
+
#
|
|
100
|
+
# class FetchCommentsJob
|
|
101
|
+
# include Sidekiq::Worker
|
|
102
|
+
# include Sidekiq::Throttled::Worker
|
|
103
|
+
#
|
|
104
|
+
# sidekiq_throttle_as :google_api
|
|
105
|
+
# end
|
|
106
|
+
#
|
|
107
|
+
# With the above configuration we ensure that there are maximum 10
|
|
108
|
+
# concurrently running jobs of FetchProfileJob or FetchCommentsJob
|
|
109
|
+
# allowed. And only 123 jobs of those are executed per hour.
|
|
110
|
+
#
|
|
111
|
+
# In other words, it will allow:
|
|
112
|
+
#
|
|
113
|
+
# - only `X` concurrent `FetchProfileJob`s
|
|
114
|
+
# - max `XX` `FetchProfileJob` per hour
|
|
115
|
+
# - only `Y` concurrent `FetchCommentsJob`s
|
|
116
|
+
# - max `YY` `FetchCommentsJob` per hour
|
|
117
|
+
#
|
|
118
|
+
# Where `(X + Y) == 10` and `(XX + YY) == 123`
|
|
119
|
+
#
|
|
120
|
+
# @see Registry.add_alias
|
|
121
|
+
# @return [void]
|
|
122
|
+
def sidekiq_throttle_as(name)
|
|
123
|
+
Registry.add_alias(self, name)
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
end
|
|
127
|
+
end
|
|
128
|
+
end
|
|
@@ -12,8 +12,12 @@ module Sidekiq
|
|
|
12
12
|
|
|
13
13
|
def key(job_args)
|
|
14
14
|
key = @base_key.dup
|
|
15
|
-
key
|
|
16
|
-
|
|
15
|
+
return key unless @key_suffix
|
|
16
|
+
|
|
17
|
+
key << ":#{@key_suffix.call(*job_args)}"
|
|
18
|
+
rescue => e
|
|
19
|
+
Sidekiq.logger.error "Failed to get key suffix: #{e}"
|
|
20
|
+
raise e
|
|
17
21
|
end
|
|
18
22
|
end
|
|
19
23
|
end
|
|
@@ -60,10 +60,9 @@ module Sidekiq
|
|
|
60
60
|
def make_strategy(strategy, name, key_suffix, options)
|
|
61
61
|
return unless options
|
|
62
62
|
|
|
63
|
-
strategy.new("throttled:#{name}",
|
|
63
|
+
strategy.new("throttled:#{name}",
|
|
64
64
|
:key_suffix => key_suffix,
|
|
65
|
-
**options
|
|
66
|
-
})
|
|
65
|
+
**options)
|
|
67
66
|
end
|
|
68
67
|
end
|
|
69
68
|
end
|
|
@@ -37,7 +37,8 @@ module Sidekiq
|
|
|
37
37
|
percentile = 100.00 * int / max
|
|
38
38
|
lvl = if 80 <= percentile then "danger"
|
|
39
39
|
elsif 60 <= percentile then "warning"
|
|
40
|
-
else
|
|
40
|
+
else
|
|
41
|
+
"success"
|
|
41
42
|
end
|
|
42
43
|
|
|
43
44
|
%(<span class="label label-#{lvl}">#{int}</span>)
|
|
@@ -61,10 +62,10 @@ module Sidekiq
|
|
|
61
62
|
|
|
62
63
|
# @return [String]
|
|
63
64
|
def humanize_integer(int)
|
|
64
|
-
digits = int.to_s.
|
|
65
|
-
str = digits.shift(digits.count % 3).join
|
|
65
|
+
digits = int.to_s.chars
|
|
66
|
+
str = digits.shift(digits.count % 3).join
|
|
66
67
|
|
|
67
|
-
str << " " << digits.shift(3).join
|
|
68
|
+
str << " " << digits.shift(3).join while digits.count.positive?
|
|
68
69
|
|
|
69
70
|
str.strip
|
|
70
71
|
end
|
|
@@ -1,128 +1,13 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
-
|
|
4
|
-
require "sidekiq/throttled/registry"
|
|
3
|
+
require "sidekiq/throttled/job"
|
|
5
4
|
|
|
6
5
|
module Sidekiq
|
|
7
6
|
module Throttled
|
|
8
|
-
#
|
|
9
|
-
#
|
|
10
|
-
#
|
|
11
|
-
#
|
|
12
|
-
|
|
13
|
-
# include Sidekiq::Worker
|
|
14
|
-
# include Sidekiq::Throttled::Worker
|
|
15
|
-
#
|
|
16
|
-
# sidkiq_options :queue => :my_queue
|
|
17
|
-
# sidekiq_throttle :threshold => { :limit => 123, :period => 1.hour }
|
|
18
|
-
#
|
|
19
|
-
# def perform
|
|
20
|
-
# # ...
|
|
21
|
-
# end
|
|
22
|
-
# end
|
|
23
|
-
#
|
|
24
|
-
# @see ClassMethods
|
|
25
|
-
module Worker
|
|
26
|
-
# Extends worker class with {ClassMethods}.
|
|
27
|
-
#
|
|
28
|
-
# @note Using `included` hook with extending worker with {ClassMethods}
|
|
29
|
-
# in order to make API inline with `include Sidekiq::Worker`.
|
|
30
|
-
#
|
|
31
|
-
# @private
|
|
32
|
-
def self.included(worker)
|
|
33
|
-
worker.send(:extend, ClassMethods)
|
|
34
|
-
end
|
|
35
|
-
|
|
36
|
-
# Helper methods added to the singleton class of destination
|
|
37
|
-
module ClassMethods
|
|
38
|
-
# Registers some strategy for the worker.
|
|
39
|
-
#
|
|
40
|
-
# @example Allow max 123 MyWorker jobs per hour
|
|
41
|
-
#
|
|
42
|
-
# class MyWorker
|
|
43
|
-
# include Sidekiq::Worker
|
|
44
|
-
# include Sidekiq::Throttled::Worker
|
|
45
|
-
#
|
|
46
|
-
# sidekiq_throttle({
|
|
47
|
-
# :threshold => { :limit => 123, :period => 1.hour }
|
|
48
|
-
# })
|
|
49
|
-
# end
|
|
50
|
-
#
|
|
51
|
-
# @example Allow max 10 concurrently running MyWorker jobs
|
|
52
|
-
#
|
|
53
|
-
# class MyWorker
|
|
54
|
-
# include Sidekiq::Worker
|
|
55
|
-
# include Sidekiq::Throttled::Worker
|
|
56
|
-
#
|
|
57
|
-
# sidekiq_throttle({
|
|
58
|
-
# :concurrency => { :limit => 10 }
|
|
59
|
-
# })
|
|
60
|
-
# end
|
|
61
|
-
#
|
|
62
|
-
# @example Allow max 10 concurrent MyWorker jobs and max 123 per hour
|
|
63
|
-
#
|
|
64
|
-
# class MyWorker
|
|
65
|
-
# include Sidekiq::Worker
|
|
66
|
-
# include Sidekiq::Throttled::Worker
|
|
67
|
-
#
|
|
68
|
-
# sidekiq_throttle({
|
|
69
|
-
# :threshold => { :limit => 123, :period => 1.hour },
|
|
70
|
-
# :concurrency => { :limit => 10 }
|
|
71
|
-
# })
|
|
72
|
-
# end
|
|
73
|
-
#
|
|
74
|
-
# @see Registry.add
|
|
75
|
-
# @return [void]
|
|
76
|
-
def sidekiq_throttle(**kwargs)
|
|
77
|
-
Registry.add(self, **kwargs)
|
|
78
|
-
end
|
|
79
|
-
|
|
80
|
-
# Adds current worker to preconfigured throttling strategy. Allows
|
|
81
|
-
# sharing same pool for multiple workers.
|
|
82
|
-
#
|
|
83
|
-
# First of all we need to create shared throttling strategy:
|
|
84
|
-
#
|
|
85
|
-
# # Create google_api throttling strategy
|
|
86
|
-
# Sidekiq::Throttled::Registry.add(:google_api, {
|
|
87
|
-
# :threshold => { :limit => 123, :period => 1.hour },
|
|
88
|
-
# :concurrency => { :limit => 10 }
|
|
89
|
-
# })
|
|
90
|
-
#
|
|
91
|
-
# Now we can assign it to our workers:
|
|
92
|
-
#
|
|
93
|
-
# class FetchProfileJob
|
|
94
|
-
# include Sidekiq::Worker
|
|
95
|
-
# include Sidekiq::Throttled::Worker
|
|
96
|
-
#
|
|
97
|
-
# sidekiq_throttle_as :google_api
|
|
98
|
-
# end
|
|
99
|
-
#
|
|
100
|
-
# class FetchCommentsJob
|
|
101
|
-
# include Sidekiq::Worker
|
|
102
|
-
# include Sidekiq::Throttled::Worker
|
|
103
|
-
#
|
|
104
|
-
# sidekiq_throttle_as :google_api
|
|
105
|
-
# end
|
|
106
|
-
#
|
|
107
|
-
# With the above configuration we ensure that there are maximum 10
|
|
108
|
-
# concurrently running jobs of FetchProfileJob or FetchCommentsJob
|
|
109
|
-
# allowed. And only 123 jobs of those are executed per hour.
|
|
110
|
-
#
|
|
111
|
-
# In other words, it will allow:
|
|
112
|
-
#
|
|
113
|
-
# - only `X` concurrent `FetchProfileJob`s
|
|
114
|
-
# - max `XX` `FetchProfileJob` per hour
|
|
115
|
-
# - only `Y` concurrent `FetchCommentsJob`s
|
|
116
|
-
# - max `YY` `FetchCommentsJob` per hour
|
|
117
|
-
#
|
|
118
|
-
# Where `(X + Y) == 10` and `(XX + YY) == 123`
|
|
119
|
-
#
|
|
120
|
-
# @see Registry.add_alias
|
|
121
|
-
# @return [void]
|
|
122
|
-
def sidekiq_throttle_as(name)
|
|
123
|
-
Registry.add_alias(self, name)
|
|
124
|
-
end
|
|
125
|
-
end
|
|
126
|
-
end
|
|
7
|
+
# A new module, Sidekiq::Job, was added in Sidekiq version 6.3.0 as a
|
|
8
|
+
# simple alias for Sidekiq::Worker as the term "worker" was considered
|
|
9
|
+
# too generic and confusing. Many people call a Sidekiq process a "worker"
|
|
10
|
+
# whereas others call the thread that executes jobs a "worker".
|
|
11
|
+
Worker = Job
|
|
127
12
|
end
|
|
128
13
|
end
|
data/lib/sidekiq/throttled.rb
CHANGED
|
@@ -9,6 +9,7 @@ require "sidekiq/throttled/communicator"
|
|
|
9
9
|
require "sidekiq/throttled/configuration"
|
|
10
10
|
require "sidekiq/throttled/queues_pauser"
|
|
11
11
|
require "sidekiq/throttled/registry"
|
|
12
|
+
require "sidekiq/throttled/job"
|
|
12
13
|
require "sidekiq/throttled/worker"
|
|
13
14
|
require "sidekiq/throttled/utils"
|
|
14
15
|
|
|
@@ -76,7 +77,7 @@ module Sidekiq
|
|
|
76
77
|
# @return [Boolean]
|
|
77
78
|
def throttled?(message)
|
|
78
79
|
message = JSON.parse message
|
|
79
|
-
job = message.fetch("class") { return false }
|
|
80
|
+
job = message.fetch("wrapped") { message.fetch("class") { return false } }
|
|
80
81
|
jid = message.fetch("jid") { return false }
|
|
81
82
|
|
|
82
83
|
preload_constant! job
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
data/sidekiq-throttled.gemspec
CHANGED
|
@@ -8,27 +8,29 @@ require "sidekiq/throttled/version"
|
|
|
8
8
|
Gem::Specification.new do |spec|
|
|
9
9
|
spec.name = "sidekiq-throttled"
|
|
10
10
|
spec.version = Sidekiq::Throttled::VERSION
|
|
11
|
-
spec.authors = ["Alexey
|
|
12
|
-
spec.email = ["
|
|
11
|
+
spec.authors = ["Alexey Zapparov"]
|
|
12
|
+
spec.email = ["alexey@zapparov.com"]
|
|
13
13
|
|
|
14
14
|
spec.summary = "Concurrency and threshold throttling for Sidekiq."
|
|
15
15
|
spec.description = "Concurrency and threshold throttling for Sidekiq."
|
|
16
|
-
spec.homepage = "https://github.com/
|
|
16
|
+
spec.homepage = "https://github.com/ixti/sidekiq-throttled"
|
|
17
17
|
spec.license = "MIT"
|
|
18
18
|
|
|
19
19
|
spec.files = `git ls-files -z`.split("\x0").reject do |f|
|
|
20
20
|
f.match %r{^(test|spec|features)/}
|
|
21
21
|
end
|
|
22
22
|
|
|
23
|
+
spec.metadata["rubygems_mfa_required"] = "true"
|
|
24
|
+
|
|
23
25
|
spec.bindir = "exe"
|
|
24
26
|
spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
|
|
25
27
|
spec.require_paths = ["lib"]
|
|
26
28
|
|
|
27
|
-
spec.required_ruby_version = "
|
|
29
|
+
spec.required_ruby_version = ">= 2.7"
|
|
28
30
|
|
|
29
31
|
spec.add_runtime_dependency "concurrent-ruby"
|
|
30
32
|
spec.add_runtime_dependency "redis-prescription"
|
|
31
33
|
spec.add_runtime_dependency "sidekiq"
|
|
32
34
|
|
|
33
|
-
spec.add_development_dependency "bundler", "
|
|
35
|
+
spec.add_development_dependency "bundler", ">= 2.0"
|
|
34
36
|
end
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: sidekiq-throttled
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.
|
|
4
|
+
version: 0.16.1
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
|
-
- Alexey
|
|
7
|
+
- Alexey Zapparov
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: exe
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2022-06-14 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: concurrent-ruby
|
|
@@ -56,34 +56,29 @@ dependencies:
|
|
|
56
56
|
name: bundler
|
|
57
57
|
requirement: !ruby/object:Gem::Requirement
|
|
58
58
|
requirements:
|
|
59
|
-
- - "
|
|
59
|
+
- - ">="
|
|
60
60
|
- !ruby/object:Gem::Version
|
|
61
61
|
version: '2.0'
|
|
62
62
|
type: :development
|
|
63
63
|
prerelease: false
|
|
64
64
|
version_requirements: !ruby/object:Gem::Requirement
|
|
65
65
|
requirements:
|
|
66
|
-
- - "
|
|
66
|
+
- - ">="
|
|
67
67
|
- !ruby/object:Gem::Version
|
|
68
68
|
version: '2.0'
|
|
69
69
|
description: Concurrency and threshold throttling for Sidekiq.
|
|
70
70
|
email:
|
|
71
|
-
-
|
|
71
|
+
- alexey@zapparov.com
|
|
72
72
|
executables: []
|
|
73
73
|
extensions: []
|
|
74
74
|
extra_rdoc_files: []
|
|
75
75
|
files:
|
|
76
76
|
- ".coveralls.yml"
|
|
77
|
+
- ".github/dependabot.yml"
|
|
77
78
|
- ".github/workflows/ci.yml"
|
|
78
79
|
- ".gitignore"
|
|
79
80
|
- ".rspec"
|
|
80
81
|
- ".rubocop.yml"
|
|
81
|
-
- ".rubocop/layout.yml"
|
|
82
|
-
- ".rubocop/lint.yml"
|
|
83
|
-
- ".rubocop/metrics.yml"
|
|
84
|
-
- ".rubocop/performance.yml"
|
|
85
|
-
- ".rubocop/rspec.yml"
|
|
86
|
-
- ".rubocop/style.yml"
|
|
87
82
|
- ".rubocop_todo.yml"
|
|
88
83
|
- ".travis.yml"
|
|
89
84
|
- ".yardopts"
|
|
@@ -94,20 +89,23 @@ files:
|
|
|
94
89
|
- LICENSE.md
|
|
95
90
|
- README.md
|
|
96
91
|
- Rakefile
|
|
97
|
-
- gemfiles/sidekiq_5.0.gemfile
|
|
98
|
-
- gemfiles/sidekiq_5.1.gemfile
|
|
99
|
-
- gemfiles/sidekiq_5.2.gemfile
|
|
100
92
|
- gemfiles/sidekiq_6.0.gemfile
|
|
101
93
|
- gemfiles/sidekiq_6.1.gemfile
|
|
94
|
+
- gemfiles/sidekiq_6.2.gemfile
|
|
95
|
+
- gemfiles/sidekiq_6.3.gemfile
|
|
96
|
+
- gemfiles/sidekiq_6.4.gemfile
|
|
97
|
+
- gemfiles/sidekiq_6.5.gemfile
|
|
102
98
|
- lib/sidekiq/throttled.rb
|
|
103
99
|
- lib/sidekiq/throttled/communicator.rb
|
|
104
100
|
- lib/sidekiq/throttled/communicator/callbacks.rb
|
|
101
|
+
- lib/sidekiq/throttled/communicator/exception_handler.rb
|
|
105
102
|
- lib/sidekiq/throttled/communicator/listener.rb
|
|
106
103
|
- lib/sidekiq/throttled/configuration.rb
|
|
107
104
|
- lib/sidekiq/throttled/errors.rb
|
|
108
105
|
- lib/sidekiq/throttled/expirable_list.rb
|
|
109
106
|
- lib/sidekiq/throttled/fetch.rb
|
|
110
107
|
- lib/sidekiq/throttled/fetch/unit_of_work.rb
|
|
108
|
+
- lib/sidekiq/throttled/job.rb
|
|
111
109
|
- lib/sidekiq/throttled/middleware.rb
|
|
112
110
|
- lib/sidekiq/throttled/patches/queue.rb
|
|
113
111
|
- lib/sidekiq/throttled/queue_name.rb
|
|
@@ -130,27 +128,34 @@ files:
|
|
|
130
128
|
- lib/sidekiq/throttled/web/summary_fix.rb
|
|
131
129
|
- lib/sidekiq/throttled/web/throttled.html.erb
|
|
132
130
|
- lib/sidekiq/throttled/worker.rb
|
|
131
|
+
- rubocop/layout.yml
|
|
132
|
+
- rubocop/lint.yml
|
|
133
|
+
- rubocop/metrics.yml
|
|
134
|
+
- rubocop/performance.yml
|
|
135
|
+
- rubocop/rspec.yml
|
|
136
|
+
- rubocop/style.yml
|
|
133
137
|
- sidekiq-throttled.gemspec
|
|
134
|
-
homepage: https://github.com/
|
|
138
|
+
homepage: https://github.com/ixti/sidekiq-throttled
|
|
135
139
|
licenses:
|
|
136
140
|
- MIT
|
|
137
|
-
metadata:
|
|
141
|
+
metadata:
|
|
142
|
+
rubygems_mfa_required: 'true'
|
|
138
143
|
post_install_message:
|
|
139
144
|
rdoc_options: []
|
|
140
145
|
require_paths:
|
|
141
146
|
- lib
|
|
142
147
|
required_ruby_version: !ruby/object:Gem::Requirement
|
|
143
148
|
requirements:
|
|
144
|
-
- - "
|
|
149
|
+
- - ">="
|
|
145
150
|
- !ruby/object:Gem::Version
|
|
146
|
-
version: '2.
|
|
151
|
+
version: '2.7'
|
|
147
152
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
|
148
153
|
requirements:
|
|
149
154
|
- - ">="
|
|
150
155
|
- !ruby/object:Gem::Version
|
|
151
156
|
version: '0'
|
|
152
157
|
requirements: []
|
|
153
|
-
rubygems_version: 3.1.
|
|
158
|
+
rubygems_version: 3.1.6
|
|
154
159
|
signing_key:
|
|
155
160
|
specification_version: 4
|
|
156
161
|
summary: Concurrency and threshold throttling for Sidekiq.
|