sidekiq-limit_fetch 4.3.2 → 4.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +3 -1
- data/.rubocop.yml +29 -0
- data/Appraisals +6 -0
- data/Gemfile +2 -0
- data/README.md +11 -1
- data/Rakefile +2 -0
- data/bench/compare.rb +17 -13
- data/demo/Gemfile +3 -2
- data/demo/Rakefile +6 -5
- data/demo/app/workers/a_worker.rb +2 -0
- data/demo/app/workers/b_worker.rb +2 -0
- data/demo/app/workers/c_worker.rb +2 -1
- data/demo/app/workers/fast_worker.rb +2 -0
- data/demo/app/workers/slow_worker.rb +2 -0
- data/demo/config/application.rb +3 -1
- data/demo/config/boot.rb +4 -2
- data/demo/config/environment.rb +3 -1
- data/demo/config/environments/development.rb +2 -0
- data/docker-compose.dev.yml +13 -0
- data/gemfiles/sidekiq_6.0.gemfile.lock +3 -4
- data/gemfiles/sidekiq_6.1.gemfile.lock +3 -4
- data/gemfiles/sidekiq_6.2.gemfile.lock +3 -4
- data/gemfiles/sidekiq_6.3.gemfile.lock +3 -4
- data/gemfiles/sidekiq_6.4.gemfile.lock +3 -4
- data/gemfiles/sidekiq_6.5.gemfile.lock +3 -4
- data/gemfiles/sidekiq_7.0.gemfile +7 -0
- data/gemfiles/sidekiq_7.0.gemfile.lock +58 -0
- data/gemfiles/sidekiq_master.gemfile.lock +3 -4
- data/lib/sidekiq/extensions/manager.rb +21 -13
- data/lib/sidekiq/extensions/queue.rb +16 -13
- data/lib/sidekiq/limit_fetch/global/monitor.rb +64 -53
- data/lib/sidekiq/limit_fetch/global/selector.rb +49 -40
- data/lib/sidekiq/limit_fetch/global/semaphore.rb +130 -123
- data/lib/sidekiq/limit_fetch/instances.rb +22 -16
- data/lib/sidekiq/limit_fetch/queues.rb +165 -124
- data/lib/sidekiq/limit_fetch/unit_of_work.rb +26 -22
- data/lib/sidekiq/limit_fetch.rb +73 -54
- data/lib/sidekiq-limit_fetch.rb +2 -0
- data/sidekiq-limit_fetch.gemspec +20 -13
- data/spec/sidekiq/extensions/manager_spec.rb +19 -0
- data/spec/sidekiq/extensions/queue_spec.rb +2 -0
- data/spec/sidekiq/limit_fetch/global/monitor_spec.rb +86 -5
- data/spec/sidekiq/limit_fetch/queues_spec.rb +34 -18
- data/spec/sidekiq/limit_fetch/semaphore_spec.rb +2 -0
- data/spec/sidekiq/limit_fetch_spec.rb +14 -4
- data/spec/spec_helper.rb +15 -4
- metadata +34 -21
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: a8b031f9486e9a497f130f2b2ed58058980cdc14985895c4cfa9ab29dbb16509
|
4
|
+
data.tar.gz: 5c5c8c2d5b144405fc5a57bf5de67dda9709939c59bdcc7870f92ae43b95de55
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 1a1cb3a5542dd9888fce9bddaabc7e8e862df7741750b9323ae637970c08be263bab2e662965b89fefb8a02f830e5f4493d46bad2e0dc6330d143cc801c2dc15
|
7
|
+
data.tar.gz: 760cc8e8bd550b7dbcd3caaa03340d9d5629f83746197254e486f07c01f52b05afb6c964afb5eb4fc4c3a95a64c1425fc358d5e806a54bf38ae4521a6ed48d06
|
data/.github/workflows/ci.yml
CHANGED
@@ -15,7 +15,7 @@ jobs:
|
|
15
15
|
fail-fast: false
|
16
16
|
matrix:
|
17
17
|
ruby: ["2.7", "3.0", "3.1"]
|
18
|
-
appraisal: ['6.0', '6.1', '6.2', '6.3', '6.4', '6.5', 'master']
|
18
|
+
appraisal: ['6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '7.0', 'master']
|
19
19
|
services:
|
20
20
|
redis:
|
21
21
|
image: redis
|
@@ -36,6 +36,8 @@ jobs:
|
|
36
36
|
bundler-cache: false
|
37
37
|
- name: Bundle for Appraisal
|
38
38
|
run: bundle
|
39
|
+
- name: Run Rubocop linting
|
40
|
+
run: bundle exec rubocop
|
39
41
|
- name: Install Appraisal dependencies
|
40
42
|
run: bundle exec appraisal sidekiq-${{ matrix.appraisal }} bundle
|
41
43
|
- name: Run tests
|
data/.rubocop.yml
ADDED
@@ -0,0 +1,29 @@
|
|
1
|
+
AllCops:
|
2
|
+
NewCops: enable
|
3
|
+
SuggestExtensions: false
|
4
|
+
TargetRubyVersion: 2.7
|
5
|
+
Exclude:
|
6
|
+
- 'gemfiles/*'
|
7
|
+
|
8
|
+
Style/Documentation:
|
9
|
+
Enabled: false
|
10
|
+
|
11
|
+
Metrics/BlockLength:
|
12
|
+
Exclude:
|
13
|
+
- 'spec/**/*.rb'
|
14
|
+
- 'demo/Rakefile'
|
15
|
+
|
16
|
+
Metrics/ModuleLength:
|
17
|
+
Exclude:
|
18
|
+
- 'lib/sidekiq/limit_fetch/queues.rb'
|
19
|
+
|
20
|
+
Metrics/ClassLength:
|
21
|
+
Exclude:
|
22
|
+
- 'lib/sidekiq/limit_fetch/global/semaphore.rb'
|
23
|
+
|
24
|
+
Naming/VariableNumber:
|
25
|
+
EnforcedStyle: snake_case
|
26
|
+
|
27
|
+
Naming/FileName:
|
28
|
+
Exclude:
|
29
|
+
- lib/sidekiq-limit_fetch.rb
|
data/Appraisals
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
appraise 'sidekiq-6.0' do
|
2
4
|
gem 'sidekiq', '~> 6.0.0'
|
3
5
|
end
|
@@ -22,6 +24,10 @@ appraise 'sidekiq-6.5' do
|
|
22
24
|
gem 'sidekiq', '~> 6.5.0'
|
23
25
|
end
|
24
26
|
|
27
|
+
appraise 'sidekiq-7.0' do
|
28
|
+
gem 'sidekiq', '~> 7.0.0'
|
29
|
+
end
|
30
|
+
|
25
31
|
appraise 'sidekiq-master' do
|
26
32
|
gem 'sidekiq', github: 'mperham/sidekiq'
|
27
33
|
end
|
data/Gemfile
CHANGED
data/README.md
CHANGED
@@ -1,6 +1,6 @@
|
|
1
1
|
## Description
|
2
2
|
|
3
|
-
*This project has been taken over by [@deanpcmad](https://github.com/deanpcmad)
|
3
|
+
*This project has been taken over by [@deanpcmad](https://github.com/deanpcmad). Original code by [@brainopia](https://github.com/brainopia).*
|
4
4
|
|
5
5
|
Sidekiq strategy to support a granular queue control – limiting, pausing, blocking, querying.
|
6
6
|
|
@@ -150,6 +150,16 @@ To use this mode you need to specify a following line in sidekiq.yml:
|
|
150
150
|
:dynamic: true
|
151
151
|
```
|
152
152
|
|
153
|
+
or
|
154
|
+
|
155
|
+
```yaml
|
156
|
+
:dynamic:
|
157
|
+
:exclude:
|
158
|
+
- excluded_queue
|
159
|
+
```
|
160
|
+
|
161
|
+
to exclude `excluded_queue` from dynamic queue
|
162
|
+
|
153
163
|
Dynamic queues will be ran at the lowest priority.
|
154
164
|
|
155
165
|
### Maintenance
|
data/Rakefile
CHANGED
data/bench/compare.rb
CHANGED
@@ -1,3 +1,5 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
require 'benchmark'
|
2
4
|
require 'sidekiq/cli'
|
3
5
|
require 'sidekiq/api'
|
@@ -9,11 +11,11 @@ limit = ARGV.shift
|
|
9
11
|
if limit
|
10
12
|
limit = nil if limit == 'nil'
|
11
13
|
|
12
|
-
|
14
|
+
$LOAD_PATH.unshift File.expand_path '../lib'
|
13
15
|
require 'sidekiq-limit_fetch'
|
14
16
|
Sidekiq::Queue['inline'].limit = limit
|
15
|
-
Sidekiq.redis {|it| it.del 'limit_fetch:probed:inline' }
|
16
|
-
Sidekiq::LimitFetch::Queues.send(:define_method, :set) {|*| }
|
17
|
+
Sidekiq.redis { |it| it.del 'limit_fetch:probed:inline' }
|
18
|
+
Sidekiq::LimitFetch::Queues.send(:define_method, :set) { |*| } # rubocop:disable Lint/EmptyBlock
|
17
19
|
end
|
18
20
|
|
19
21
|
Sidekiq::Queue.new('inline').clear
|
@@ -22,8 +24,8 @@ class FastJob
|
|
22
24
|
include Sidekiq::Worker
|
23
25
|
sidekiq_options queue: :inline
|
24
26
|
|
25
|
-
def perform(
|
26
|
-
puts "job N#{
|
27
|
+
def perform(index)
|
28
|
+
puts "job N#{index} is finished"
|
27
29
|
end
|
28
30
|
end
|
29
31
|
|
@@ -36,17 +38,19 @@ class FinishJob
|
|
36
38
|
end
|
37
39
|
end
|
38
40
|
|
39
|
-
total.times {|i| FastJob.perform_async i+1 }
|
41
|
+
total.times { |i| FastJob.perform_async i + 1 }
|
40
42
|
FinishJob.perform_async
|
41
43
|
|
42
44
|
Sidekiq::CLI.instance.tap do |cli|
|
43
|
-
%w
|
45
|
+
%w[validate! boot_system].each { |stub| cli.define_singleton_method(stub) {} } # rubocop:disable Lint/EmptyBlock
|
44
46
|
cli.parse ['-q inline', '-q other', "-c #{concurrency}"]
|
45
47
|
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
48
|
+
# rubocop:disable Lint/RescueException
|
49
|
+
# rubocop:disable Lint/SuppressedException
|
50
|
+
puts Benchmark.measure do
|
51
|
+
cli.run
|
52
|
+
rescue Exception
|
53
|
+
end
|
54
|
+
# rubocop:enable Lint/SuppressedException
|
55
|
+
# rubocop:enable Lint/RescueException
|
52
56
|
end
|
data/demo/Gemfile
CHANGED
data/demo/Rakefile
CHANGED
@@ -1,4 +1,6 @@
|
|
1
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require File.expand_path('config/application', __dir__)
|
2
4
|
Demo::Application.load_tasks
|
3
5
|
|
4
6
|
namespace :demo do
|
@@ -63,7 +65,7 @@ namespace :demo do
|
|
63
65
|
end
|
64
66
|
def with_sidekiq_config(config)
|
65
67
|
whitespace_offset = config[/\A */].size
|
66
|
-
config.gsub!
|
68
|
+
config.gsub!(/^ {#{whitespace_offset}}/, '')
|
67
69
|
|
68
70
|
puts "=> Use sidekiq config:\n#{config}"
|
69
71
|
File.write 'config/sidekiq.yml', config
|
@@ -85,8 +87,8 @@ namespace :demo do
|
|
85
87
|
require 'sidekiq/cli'
|
86
88
|
cli = Sidekiq::CLI.instance
|
87
89
|
|
88
|
-
%w
|
89
|
-
cli.define_singleton_method(stub) {}
|
90
|
+
%w[validate! boot_system].each do |stub|
|
91
|
+
cli.define_singleton_method(stub) {} # rubocop:disable Lint/EmptyBlock
|
90
92
|
end
|
91
93
|
|
92
94
|
with_sidekiq_config options[:config] do
|
@@ -96,4 +98,3 @@ namespace :demo do
|
|
96
98
|
cli.run
|
97
99
|
end
|
98
100
|
end
|
99
|
-
|
data/demo/config/application.rb
CHANGED
data/demo/config/boot.rb
CHANGED
@@ -1,2 +1,4 @@
|
|
1
|
-
|
2
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
ENV['BUNDLE_GEMFILE'] ||= File.expand_path('../Gemfile', __dir__)
|
4
|
+
require 'bundler/setup' if File.exist?(ENV['BUNDLE_GEMFILE'])
|
data/demo/config/environment.rb
CHANGED
@@ -1,9 +1,8 @@
|
|
1
1
|
PATH
|
2
2
|
remote: ..
|
3
3
|
specs:
|
4
|
-
sidekiq-limit_fetch (4.
|
5
|
-
|
6
|
-
sidekiq (>= 4)
|
4
|
+
sidekiq-limit_fetch (4.4.0)
|
5
|
+
sidekiq (>= 6)
|
7
6
|
|
8
7
|
GEM
|
9
8
|
remote: https://rubygems.org/
|
@@ -53,4 +52,4 @@ DEPENDENCIES
|
|
53
52
|
sidekiq-limit_fetch!
|
54
53
|
|
55
54
|
BUNDLED WITH
|
56
|
-
2.3.
|
55
|
+
2.3.5
|
@@ -1,9 +1,8 @@
|
|
1
1
|
PATH
|
2
2
|
remote: ..
|
3
3
|
specs:
|
4
|
-
sidekiq-limit_fetch (4.
|
5
|
-
|
6
|
-
sidekiq (>= 4)
|
4
|
+
sidekiq-limit_fetch (4.4.0)
|
5
|
+
sidekiq (>= 6)
|
7
6
|
|
8
7
|
GEM
|
9
8
|
remote: https://rubygems.org/
|
@@ -50,4 +49,4 @@ DEPENDENCIES
|
|
50
49
|
sidekiq-limit_fetch!
|
51
50
|
|
52
51
|
BUNDLED WITH
|
53
|
-
2.3.
|
52
|
+
2.3.5
|
@@ -1,9 +1,8 @@
|
|
1
1
|
PATH
|
2
2
|
remote: ..
|
3
3
|
specs:
|
4
|
-
sidekiq-limit_fetch (4.
|
5
|
-
|
6
|
-
sidekiq (>= 4)
|
4
|
+
sidekiq-limit_fetch (4.4.0)
|
5
|
+
sidekiq (>= 6)
|
7
6
|
|
8
7
|
GEM
|
9
8
|
remote: https://rubygems.org/
|
@@ -50,4 +49,4 @@ DEPENDENCIES
|
|
50
49
|
sidekiq-limit_fetch!
|
51
50
|
|
52
51
|
BUNDLED WITH
|
53
|
-
2.3.
|
52
|
+
2.3.5
|
@@ -1,9 +1,8 @@
|
|
1
1
|
PATH
|
2
2
|
remote: ..
|
3
3
|
specs:
|
4
|
-
sidekiq-limit_fetch (4.
|
5
|
-
|
6
|
-
sidekiq (>= 4)
|
4
|
+
sidekiq-limit_fetch (4.4.0)
|
5
|
+
sidekiq (>= 6)
|
7
6
|
|
8
7
|
GEM
|
9
8
|
remote: https://rubygems.org/
|
@@ -50,4 +49,4 @@ DEPENDENCIES
|
|
50
49
|
sidekiq-limit_fetch!
|
51
50
|
|
52
51
|
BUNDLED WITH
|
53
|
-
2.3.
|
52
|
+
2.3.5
|
@@ -1,9 +1,8 @@
|
|
1
1
|
PATH
|
2
2
|
remote: ..
|
3
3
|
specs:
|
4
|
-
sidekiq-limit_fetch (4.
|
5
|
-
|
6
|
-
sidekiq (>= 4)
|
4
|
+
sidekiq-limit_fetch (4.4.0)
|
5
|
+
sidekiq (>= 6)
|
7
6
|
|
8
7
|
GEM
|
9
8
|
remote: https://rubygems.org/
|
@@ -50,4 +49,4 @@ DEPENDENCIES
|
|
50
49
|
sidekiq-limit_fetch!
|
51
50
|
|
52
51
|
BUNDLED WITH
|
53
|
-
2.3.
|
52
|
+
2.3.5
|
@@ -1,9 +1,8 @@
|
|
1
1
|
PATH
|
2
2
|
remote: ..
|
3
3
|
specs:
|
4
|
-
sidekiq-limit_fetch (4.
|
5
|
-
|
6
|
-
sidekiq (>= 4)
|
4
|
+
sidekiq-limit_fetch (4.4.0)
|
5
|
+
sidekiq (>= 6)
|
7
6
|
|
8
7
|
GEM
|
9
8
|
remote: https://rubygems.org/
|
@@ -50,4 +49,4 @@ DEPENDENCIES
|
|
50
49
|
sidekiq-limit_fetch!
|
51
50
|
|
52
51
|
BUNDLED WITH
|
53
|
-
2.3.
|
52
|
+
2.3.5
|
@@ -0,0 +1,58 @@
|
|
1
|
+
PATH
|
2
|
+
remote: ..
|
3
|
+
specs:
|
4
|
+
sidekiq-limit_fetch (4.4.0)
|
5
|
+
sidekiq (>= 6)
|
6
|
+
|
7
|
+
GEM
|
8
|
+
remote: https://rubygems.org/
|
9
|
+
specs:
|
10
|
+
appraisal (2.4.1)
|
11
|
+
bundler
|
12
|
+
rake
|
13
|
+
thor (>= 0.14.0)
|
14
|
+
concurrent-ruby (1.1.10)
|
15
|
+
connection_pool (2.3.0)
|
16
|
+
diff-lcs (1.5.0)
|
17
|
+
rack (3.0.2)
|
18
|
+
rake (13.0.6)
|
19
|
+
redis (5.0.5)
|
20
|
+
redis-client (>= 0.9.0)
|
21
|
+
redis-client (0.11.2)
|
22
|
+
connection_pool
|
23
|
+
redis-namespace (1.10.0)
|
24
|
+
redis (>= 4)
|
25
|
+
rspec (3.12.0)
|
26
|
+
rspec-core (~> 3.12.0)
|
27
|
+
rspec-expectations (~> 3.12.0)
|
28
|
+
rspec-mocks (~> 3.12.0)
|
29
|
+
rspec-core (3.12.0)
|
30
|
+
rspec-support (~> 3.12.0)
|
31
|
+
rspec-expectations (3.12.1)
|
32
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
33
|
+
rspec-support (~> 3.12.0)
|
34
|
+
rspec-mocks (3.12.1)
|
35
|
+
diff-lcs (>= 1.2.0, < 2.0)
|
36
|
+
rspec-support (~> 3.12.0)
|
37
|
+
rspec-support (3.12.0)
|
38
|
+
sidekiq (7.0.2)
|
39
|
+
concurrent-ruby (< 2)
|
40
|
+
connection_pool (>= 2.3.0)
|
41
|
+
rack (>= 2.2.4)
|
42
|
+
redis-client (>= 0.11.0)
|
43
|
+
thor (1.2.1)
|
44
|
+
|
45
|
+
PLATFORMS
|
46
|
+
x86_64-darwin-21
|
47
|
+
x86_64-linux
|
48
|
+
|
49
|
+
DEPENDENCIES
|
50
|
+
appraisal
|
51
|
+
rake
|
52
|
+
redis-namespace (~> 1.5, >= 1.5.2)
|
53
|
+
rspec
|
54
|
+
sidekiq (~> 7.0.0)
|
55
|
+
sidekiq-limit_fetch!
|
56
|
+
|
57
|
+
BUNDLED WITH
|
58
|
+
2.3.5
|
@@ -10,9 +10,8 @@ GIT
|
|
10
10
|
PATH
|
11
11
|
remote: ..
|
12
12
|
specs:
|
13
|
-
sidekiq-limit_fetch (4.
|
14
|
-
|
15
|
-
sidekiq (>= 4)
|
13
|
+
sidekiq-limit_fetch (4.4.0)
|
14
|
+
sidekiq (>= 6)
|
16
15
|
|
17
16
|
GEM
|
18
17
|
remote: https://rubygems.org/
|
@@ -55,4 +54,4 @@ DEPENDENCIES
|
|
55
54
|
sidekiq-limit_fetch!
|
56
55
|
|
57
56
|
BUNDLED WITH
|
58
|
-
2.3.
|
57
|
+
2.3.5
|
@@ -1,17 +1,25 @@
|
|
1
|
-
|
2
|
-
|
3
|
-
|
4
|
-
|
5
|
-
|
6
|
-
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
module Sidekiq
|
4
|
+
class Manager
|
5
|
+
module InitLimitFetch
|
6
|
+
def initialize(capsule_or_options)
|
7
|
+
if Sidekiq::LimitFetch.post_7?
|
8
|
+
capsule_or_options.config[:fetch_class] = Sidekiq::LimitFetch
|
9
|
+
else
|
10
|
+
capsule_or_options[:fetch] = Sidekiq::LimitFetch
|
11
|
+
end
|
12
|
+
super
|
13
|
+
end
|
7
14
|
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
15
|
+
def start
|
16
|
+
# In sidekiq 6.5.0 the variable @options has been renamed to @config
|
17
|
+
Sidekiq::LimitFetch::Queues.start @options || @config
|
18
|
+
Sidekiq::LimitFetch::Global::Monitor.start!
|
19
|
+
super
|
20
|
+
end
|
13
21
|
end
|
14
|
-
end
|
15
22
|
|
16
|
-
|
23
|
+
prepend InitLimitFetch
|
24
|
+
end
|
17
25
|
end
|
@@ -1,21 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
module Sidekiq
|
2
4
|
class Queue
|
3
|
-
extend
|
5
|
+
extend Forwardable
|
6
|
+
extend LimitFetch::Instances
|
4
7
|
attr_reader :rname
|
5
8
|
|
6
9
|
def_delegators :lock,
|
7
|
-
|
8
|
-
|
9
|
-
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
|
10
|
+
:limit, :limit=, :limit_changed?,
|
11
|
+
:process_limit, :process_limit=,
|
12
|
+
:acquire, :release,
|
13
|
+
:pause, :pause_for_ms, :unpause,
|
14
|
+
:block, :unblock,
|
15
|
+
:paused?, :blocking?,
|
16
|
+
:unblocked, :block_except,
|
17
|
+
:probed, :busy,
|
18
|
+
:increase_busy, :decrease_busy,
|
19
|
+
:local_busy?, :explain,
|
20
|
+
:remove_locks_except!,
|
21
|
+
:clear_limits
|
19
22
|
|
20
23
|
def lock
|
21
24
|
@lock ||= LimitFetch::Global::Semaphore.new name
|