sidekiq-limit_fetch 4.3.2 → 4.4.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +3 -1
- data/.rubocop.yml +29 -0
- data/Appraisals +6 -0
- data/Gemfile +2 -0
- data/README.md +11 -1
- data/Rakefile +2 -0
- data/bench/compare.rb +17 -13
- data/demo/Gemfile +3 -2
- data/demo/Rakefile +6 -5
- data/demo/app/workers/a_worker.rb +2 -0
- data/demo/app/workers/b_worker.rb +2 -0
- data/demo/app/workers/c_worker.rb +2 -1
- data/demo/app/workers/fast_worker.rb +2 -0
- data/demo/app/workers/slow_worker.rb +2 -0
- data/demo/config/application.rb +3 -1
- data/demo/config/boot.rb +4 -2
- data/demo/config/environment.rb +3 -1
- data/demo/config/environments/development.rb +2 -0
- data/docker-compose.dev.yml +13 -0
- data/gemfiles/sidekiq_6.0.gemfile.lock +3 -4
- data/gemfiles/sidekiq_6.1.gemfile.lock +3 -4
- data/gemfiles/sidekiq_6.2.gemfile.lock +3 -4
- data/gemfiles/sidekiq_6.3.gemfile.lock +3 -4
- data/gemfiles/sidekiq_6.4.gemfile.lock +3 -4
- data/gemfiles/sidekiq_6.5.gemfile.lock +3 -4
- data/gemfiles/sidekiq_7.0.gemfile +7 -0
- data/gemfiles/sidekiq_7.0.gemfile.lock +58 -0
- data/gemfiles/sidekiq_master.gemfile.lock +3 -4
- data/lib/sidekiq/extensions/manager.rb +21 -13
- data/lib/sidekiq/extensions/queue.rb +16 -13
- data/lib/sidekiq/limit_fetch/global/monitor.rb +64 -53
- data/lib/sidekiq/limit_fetch/global/selector.rb +49 -40
- data/lib/sidekiq/limit_fetch/global/semaphore.rb +130 -123
- data/lib/sidekiq/limit_fetch/instances.rb +22 -16
- data/lib/sidekiq/limit_fetch/queues.rb +165 -124
- data/lib/sidekiq/limit_fetch/unit_of_work.rb +26 -22
- data/lib/sidekiq/limit_fetch.rb +73 -54
- data/lib/sidekiq-limit_fetch.rb +2 -0
- data/sidekiq-limit_fetch.gemspec +20 -13
- data/spec/sidekiq/extensions/manager_spec.rb +19 -0
- data/spec/sidekiq/extensions/queue_spec.rb +2 -0
- data/spec/sidekiq/limit_fetch/global/monitor_spec.rb +86 -5
- data/spec/sidekiq/limit_fetch/queues_spec.rb +34 -18
- data/spec/sidekiq/limit_fetch/semaphore_spec.rb +2 -0
- data/spec/sidekiq/limit_fetch_spec.rb +14 -4
- data/spec/spec_helper.rb +15 -4
- metadata +34 -21
@@ -1,22 +1,24 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
RSpec.describe Sidekiq::LimitFetch::Global::Monitor do
|
2
4
|
let(:monitor) { described_class.start! ttl, timeout }
|
3
5
|
let(:ttl) { 1 }
|
4
6
|
let(:queue) { Sidekiq::Queue[name] }
|
5
7
|
let(:name) { 'default' }
|
8
|
+
let(:timeout) { 0.5 }
|
6
9
|
|
7
|
-
before { monitor }
|
8
10
|
after { monitor.kill }
|
9
11
|
|
10
12
|
context 'old locks' do
|
11
|
-
|
13
|
+
before { monitor }
|
12
14
|
|
13
15
|
it 'should remove invalidated old locks' do
|
14
16
|
2.times { queue.acquire }
|
15
|
-
sleep 2
|
17
|
+
sleep ttl * 2
|
16
18
|
expect(queue.probed).to eq 2
|
17
19
|
|
18
20
|
allow(described_class).to receive(:update_heartbeat)
|
19
|
-
sleep 2
|
21
|
+
sleep ttl * 2
|
20
22
|
expect(queue.probed).to eq 0
|
21
23
|
end
|
22
24
|
|
@@ -26,8 +28,87 @@ RSpec.describe Sidekiq::LimitFetch::Global::Monitor do
|
|
26
28
|
Sidekiq.redis do |it|
|
27
29
|
it.del Sidekiq::LimitFetch::Global::Monitor::PROCESS_SET
|
28
30
|
end
|
29
|
-
sleep 2
|
31
|
+
sleep ttl * 2
|
30
32
|
expect(queue.probed).to eq 0
|
31
33
|
end
|
32
34
|
end
|
35
|
+
|
36
|
+
context 'dynamic queue' do
|
37
|
+
let(:limits) do
|
38
|
+
{
|
39
|
+
'queue1' => 3,
|
40
|
+
'queue2' => 3
|
41
|
+
}
|
42
|
+
end
|
43
|
+
let(:queues) { %w[queue1 queue2] }
|
44
|
+
let(:queue) { Sidekiq::LimitFetch::Queues }
|
45
|
+
|
46
|
+
let(:config) { Sidekiq::Config.new(options) }
|
47
|
+
let(:capsule) do
|
48
|
+
config.capsule('default') do |cap|
|
49
|
+
cap.concurrency = 1
|
50
|
+
cap.queues = config[:queues]
|
51
|
+
end
|
52
|
+
end
|
53
|
+
|
54
|
+
let(:capsule_or_options) do
|
55
|
+
Sidekiq::LimitFetch.post_7? ? capsule : options
|
56
|
+
end
|
57
|
+
|
58
|
+
context 'without excluded queue' do
|
59
|
+
let(:options) do
|
60
|
+
{
|
61
|
+
limits: limits,
|
62
|
+
queues: queues,
|
63
|
+
dynamic: true
|
64
|
+
}
|
65
|
+
end
|
66
|
+
|
67
|
+
it 'should add dynamic queue' do
|
68
|
+
queue.start(capsule_or_options)
|
69
|
+
monitor
|
70
|
+
|
71
|
+
expect(queue.instance_variable_get(:@queues)).not_to include('queue3')
|
72
|
+
|
73
|
+
Sidekiq.redis do |it|
|
74
|
+
it.sadd 'queues', 'queue3'
|
75
|
+
end
|
76
|
+
|
77
|
+
sleep ttl * 2
|
78
|
+
expect(queue.instance_variable_get(:@queues)).to include('queue3')
|
79
|
+
|
80
|
+
Sidekiq.redis do |it|
|
81
|
+
it.srem 'queues', 'queue3'
|
82
|
+
end
|
83
|
+
end
|
84
|
+
end
|
85
|
+
|
86
|
+
context 'with excluded queue' do
|
87
|
+
let(:options) do
|
88
|
+
{
|
89
|
+
limits: limits,
|
90
|
+
queues: queues,
|
91
|
+
dynamic: { exclude: ['queue4'] }
|
92
|
+
}
|
93
|
+
end
|
94
|
+
|
95
|
+
it 'should exclude excluded dynamic queue' do
|
96
|
+
queue.start(capsule_or_options)
|
97
|
+
monitor
|
98
|
+
|
99
|
+
expect(queue.instance_variable_get(:@queues)).not_to include('queue4')
|
100
|
+
|
101
|
+
Sidekiq.redis do |it|
|
102
|
+
it.sadd 'queues', 'queue4'
|
103
|
+
end
|
104
|
+
|
105
|
+
sleep ttl * 2
|
106
|
+
expect(queue.instance_variable_get(:@queues)).not_to include('queue4')
|
107
|
+
|
108
|
+
Sidekiq.redis do |it|
|
109
|
+
it.srem 'queues', 'queue4'
|
110
|
+
end
|
111
|
+
end
|
112
|
+
end
|
113
|
+
end
|
33
114
|
end
|
@@ -1,19 +1,35 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
RSpec.describe Sidekiq::LimitFetch::Queues do
|
2
4
|
let(:queues) { %w[queue1 queue2] }
|
3
|
-
let(:limits) {{ 'queue1' => 3 }}
|
5
|
+
let(:limits) { { 'queue1' => 3 } }
|
4
6
|
let(:strict) { true }
|
5
|
-
let(:blocking) {}
|
6
|
-
let(:process_limits) {{ 'queue2' => 3 }}
|
7
|
+
let(:blocking) { nil }
|
8
|
+
let(:process_limits) { { 'queue2' => 3 } }
|
7
9
|
|
8
10
|
let(:options) do
|
9
|
-
{ queues:
|
10
|
-
limits:
|
11
|
-
strict:
|
11
|
+
{ queues: queues,
|
12
|
+
limits: limits,
|
13
|
+
strict: strict,
|
12
14
|
blocking: blocking,
|
13
15
|
process_limits: process_limits }
|
14
16
|
end
|
15
17
|
|
16
|
-
|
18
|
+
let(:config) { Sidekiq::Config.new(options) }
|
19
|
+
let(:capsule) do
|
20
|
+
config.capsule('default') do |cap|
|
21
|
+
cap.concurrency = 1
|
22
|
+
cap.queues = config[:queues]
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
let(:capsule_or_options) do
|
27
|
+
Sidekiq::LimitFetch.post_7? ? capsule : options
|
28
|
+
end
|
29
|
+
|
30
|
+
before do
|
31
|
+
subject.start(capsule_or_options)
|
32
|
+
end
|
17
33
|
|
18
34
|
def in_thread(&block)
|
19
35
|
thr = Thread.new(&block)
|
@@ -51,36 +67,36 @@ RSpec.describe Sidekiq::LimitFetch::Queues do
|
|
51
67
|
end
|
52
68
|
|
53
69
|
it 'should release queues' do
|
54
|
-
in_thread
|
70
|
+
in_thread do
|
55
71
|
subject.acquire
|
56
72
|
subject.release_except nil
|
57
|
-
|
73
|
+
end
|
58
74
|
expect(Sidekiq::Queue['queue1'].probed).to eq 0
|
59
75
|
expect(Sidekiq::Queue['queue2'].probed).to eq 0
|
60
76
|
end
|
61
77
|
|
62
78
|
it 'should release queues except selected' do
|
63
|
-
in_thread
|
79
|
+
in_thread do
|
64
80
|
subject.acquire
|
65
81
|
subject.release_except 'queue:queue1'
|
66
|
-
|
82
|
+
end
|
67
83
|
expect(Sidekiq::Queue['queue1'].probed).to eq 1
|
68
84
|
expect(Sidekiq::Queue['queue2'].probed).to eq 0
|
69
85
|
end
|
70
86
|
|
71
87
|
it 'should release when no queues was acquired' do
|
72
|
-
queues.each {|name| Sidekiq::Queue[name].pause }
|
73
|
-
in_thread
|
88
|
+
queues.each { |name| Sidekiq::Queue[name].pause }
|
89
|
+
in_thread do
|
74
90
|
subject.acquire
|
75
91
|
expect { subject.release_except nil }.not_to raise_exception
|
76
|
-
|
92
|
+
end
|
77
93
|
end
|
78
94
|
|
79
95
|
context 'blocking' do
|
80
|
-
let(:blocking) { %w
|
96
|
+
let(:blocking) { %w[queue1] }
|
81
97
|
|
82
98
|
it 'should acquire blocking queues' do
|
83
|
-
3.times { in_thread { subject.acquire
|
99
|
+
3.times { in_thread { subject.acquire } }
|
84
100
|
expect(Sidekiq::Queue['queue1'].probed).to eq 3
|
85
101
|
expect(Sidekiq::Queue['queue2'].probed).to eq 1
|
86
102
|
end
|
@@ -101,11 +117,11 @@ RSpec.describe Sidekiq::LimitFetch::Queues do
|
|
101
117
|
let(:strict) { false }
|
102
118
|
|
103
119
|
it 'should retrieve weighted queues' do
|
104
|
-
expect(subject.ordered_queues).to match_array(%w
|
120
|
+
expect(subject.ordered_queues).to match_array(%w[queue1 queue2])
|
105
121
|
end
|
106
122
|
end
|
107
123
|
|
108
124
|
it 'with strict flag should retrieve strictly ordered queues' do
|
109
|
-
expect(subject.ordered_queues).to eq %w
|
125
|
+
expect(subject.ordered_queues).to eq %w[queue1 queue2]
|
110
126
|
end
|
111
127
|
end
|
@@ -1,12 +1,22 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
1
3
|
Thread.abort_on_exception = true
|
2
4
|
|
3
5
|
RSpec.describe Sidekiq::LimitFetch do
|
4
|
-
let(:options) {{ queues: queues, limits: limits }}
|
5
|
-
let(:queues) { %w
|
6
|
-
let(:limits) {{ 'queue1' => 1, 'queue2' => 2 }}
|
6
|
+
let(:options) { { queues: queues, limits: limits } }
|
7
|
+
let(:queues) { %w[queue1 queue1 queue2 queue2] }
|
8
|
+
let(:limits) { { 'queue1' => 1, 'queue2' => 2 } }
|
9
|
+
let(:config) { Sidekiq::Config.new(options) }
|
10
|
+
let(:capsule) do
|
11
|
+
config.capsule('default') do |cap|
|
12
|
+
cap.concurrency = 1
|
13
|
+
cap.queues = config[:queues]
|
14
|
+
end
|
15
|
+
end
|
16
|
+
let(:capsule_or_config) { Sidekiq::LimitFetch.post_7? ? capsule : options }
|
7
17
|
|
8
18
|
before do
|
9
|
-
subject::Queues.start
|
19
|
+
subject::Queues.start(capsule_or_config)
|
10
20
|
|
11
21
|
Sidekiq.redis do |it|
|
12
22
|
it.del 'queue:queue1'
|
data/spec/spec_helper.rb
CHANGED
@@ -1,7 +1,18 @@
|
|
1
|
+
# frozen_string_literal: true
|
2
|
+
|
3
|
+
require 'simplecov'
|
4
|
+
SimpleCov.start
|
5
|
+
|
1
6
|
require 'sidekiq/limit_fetch'
|
2
7
|
|
3
|
-
Sidekiq.
|
4
|
-
Sidekiq.
|
8
|
+
if Sidekiq::LimitFetch.post_7?
|
9
|
+
Sidekiq.configure_embed do |config|
|
10
|
+
config.logger = nil
|
11
|
+
end
|
12
|
+
else
|
13
|
+
Sidekiq.logger = nil
|
14
|
+
Sidekiq.redis = { namespace: ENV.fetch('namespace', nil) }
|
15
|
+
end
|
5
16
|
|
6
17
|
RSpec.configure do |config|
|
7
18
|
config.order = :random
|
@@ -10,7 +21,7 @@ RSpec.configure do |config|
|
|
10
21
|
config.before do
|
11
22
|
Sidekiq::Queue.reset_instances!
|
12
23
|
Sidekiq.redis do |it|
|
13
|
-
clean_redis =
|
24
|
+
clean_redis = lambda do |queue|
|
14
25
|
it.pipelined do |pipeline|
|
15
26
|
pipeline.del "limit_fetch:limit:#{queue}"
|
16
27
|
pipeline.del "limit_fetch:process_limit:#{queue}"
|
@@ -22,7 +33,7 @@ RSpec.configure do |config|
|
|
22
33
|
end
|
23
34
|
|
24
35
|
clean_redis.call(name) if defined?(name)
|
25
|
-
queues.each(&clean_redis) if defined?(queues)
|
36
|
+
queues.each(&clean_redis) if defined?(queues) && queues.is_a?(Array)
|
26
37
|
end
|
27
38
|
end
|
28
39
|
end
|
metadata
CHANGED
@@ -1,15 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: sidekiq-limit_fetch
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 4.
|
4
|
+
version: 4.4.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Dean Perry
|
8
|
-
- brainopia
|
9
8
|
autorequire:
|
10
9
|
bindir: bin
|
11
10
|
cert_chain: []
|
12
|
-
date:
|
11
|
+
date: 2023-01-22 00:00:00.000000000 Z
|
13
12
|
dependencies:
|
14
13
|
- !ruby/object:Gem::Dependency
|
15
14
|
name: sidekiq
|
@@ -17,28 +16,42 @@ dependencies:
|
|
17
16
|
requirements:
|
18
17
|
- - ">="
|
19
18
|
- !ruby/object:Gem::Version
|
20
|
-
version: '
|
19
|
+
version: '6'
|
21
20
|
type: :runtime
|
22
21
|
prerelease: false
|
23
22
|
version_requirements: !ruby/object:Gem::Requirement
|
24
23
|
requirements:
|
25
24
|
- - ">="
|
26
25
|
- !ruby/object:Gem::Version
|
27
|
-
version: '
|
26
|
+
version: '6'
|
28
27
|
- !ruby/object:Gem::Dependency
|
29
|
-
name:
|
28
|
+
name: appraisal
|
30
29
|
requirement: !ruby/object:Gem::Requirement
|
31
30
|
requirements:
|
32
31
|
- - ">="
|
33
32
|
- !ruby/object:Gem::Version
|
34
|
-
version:
|
35
|
-
type: :
|
33
|
+
version: '0'
|
34
|
+
type: :development
|
35
|
+
prerelease: false
|
36
|
+
version_requirements: !ruby/object:Gem::Requirement
|
37
|
+
requirements:
|
38
|
+
- - ">="
|
39
|
+
- !ruby/object:Gem::Version
|
40
|
+
version: '0'
|
41
|
+
- !ruby/object:Gem::Dependency
|
42
|
+
name: rake
|
43
|
+
requirement: !ruby/object:Gem::Requirement
|
44
|
+
requirements:
|
45
|
+
- - ">="
|
46
|
+
- !ruby/object:Gem::Version
|
47
|
+
version: '0'
|
48
|
+
type: :development
|
36
49
|
prerelease: false
|
37
50
|
version_requirements: !ruby/object:Gem::Requirement
|
38
51
|
requirements:
|
39
52
|
- - ">="
|
40
53
|
- !ruby/object:Gem::Version
|
41
|
-
version:
|
54
|
+
version: '0'
|
42
55
|
- !ruby/object:Gem::Dependency
|
43
56
|
name: redis-namespace
|
44
57
|
requirement: !ruby/object:Gem::Requirement
|
@@ -60,7 +73,7 @@ dependencies:
|
|
60
73
|
- !ruby/object:Gem::Version
|
61
74
|
version: 1.5.2
|
62
75
|
- !ruby/object:Gem::Dependency
|
63
|
-
name:
|
76
|
+
name: rspec
|
64
77
|
requirement: !ruby/object:Gem::Requirement
|
65
78
|
requirements:
|
66
79
|
- - ">="
|
@@ -74,7 +87,7 @@ dependencies:
|
|
74
87
|
- !ruby/object:Gem::Version
|
75
88
|
version: '0'
|
76
89
|
- !ruby/object:Gem::Dependency
|
77
|
-
name:
|
90
|
+
name: rubocop
|
78
91
|
requirement: !ruby/object:Gem::Requirement
|
79
92
|
requirements:
|
80
93
|
- - ">="
|
@@ -88,7 +101,7 @@ dependencies:
|
|
88
101
|
- !ruby/object:Gem::Version
|
89
102
|
version: '0'
|
90
103
|
- !ruby/object:Gem::Dependency
|
91
|
-
name:
|
104
|
+
name: simplecov
|
92
105
|
requirement: !ruby/object:Gem::Requirement
|
93
106
|
requirements:
|
94
107
|
- - ">="
|
@@ -111,6 +124,7 @@ files:
|
|
111
124
|
- ".github/workflows/ci.yml"
|
112
125
|
- ".gitignore"
|
113
126
|
- ".rspec"
|
127
|
+
- ".rubocop.yml"
|
114
128
|
- Appraisals
|
115
129
|
- CHANGELOG.md
|
116
130
|
- Gemfile
|
@@ -130,6 +144,7 @@ files:
|
|
130
144
|
- demo/config/boot.rb
|
131
145
|
- demo/config/environment.rb
|
132
146
|
- demo/config/environments/development.rb
|
147
|
+
- docker-compose.dev.yml
|
133
148
|
- gemfiles/sidekiq_6.0.gemfile
|
134
149
|
- gemfiles/sidekiq_6.0.gemfile.lock
|
135
150
|
- gemfiles/sidekiq_6.1.gemfile
|
@@ -142,6 +157,8 @@ files:
|
|
142
157
|
- gemfiles/sidekiq_6.4.gemfile.lock
|
143
158
|
- gemfiles/sidekiq_6.5.gemfile
|
144
159
|
- gemfiles/sidekiq_6.5.gemfile.lock
|
160
|
+
- gemfiles/sidekiq_7.0.gemfile
|
161
|
+
- gemfiles/sidekiq_7.0.gemfile.lock
|
145
162
|
- gemfiles/sidekiq_master.gemfile
|
146
163
|
- gemfiles/sidekiq_master.gemfile.lock
|
147
164
|
- lib/sidekiq-limit_fetch.rb
|
@@ -155,6 +172,7 @@ files:
|
|
155
172
|
- lib/sidekiq/limit_fetch/queues.rb
|
156
173
|
- lib/sidekiq/limit_fetch/unit_of_work.rb
|
157
174
|
- sidekiq-limit_fetch.gemspec
|
175
|
+
- spec/sidekiq/extensions/manager_spec.rb
|
158
176
|
- spec/sidekiq/extensions/queue_spec.rb
|
159
177
|
- spec/sidekiq/limit_fetch/global/monitor_spec.rb
|
160
178
|
- spec/sidekiq/limit_fetch/queues_spec.rb
|
@@ -168,6 +186,7 @@ metadata:
|
|
168
186
|
homepage_uri: https://github.com/deanpcmad/sidekiq-limit_fetch
|
169
187
|
source_code_uri: https://github.com/deanpcmad/sidekiq-limit_fetch
|
170
188
|
changelog_uri: https://github.com/deanpcmad/sidekiq-limit_fetch/blob/master/CHANGELOG.md
|
189
|
+
rubygems_mfa_required: 'true'
|
171
190
|
post_install_message:
|
172
191
|
rdoc_options: []
|
173
192
|
require_paths:
|
@@ -176,21 +195,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
|
|
176
195
|
requirements:
|
177
196
|
- - ">="
|
178
197
|
- !ruby/object:Gem::Version
|
179
|
-
version:
|
198
|
+
version: 2.7.0
|
180
199
|
required_rubygems_version: !ruby/object:Gem::Requirement
|
181
200
|
requirements:
|
182
201
|
- - ">="
|
183
202
|
- !ruby/object:Gem::Version
|
184
203
|
version: '0'
|
185
204
|
requirements: []
|
186
|
-
rubygems_version: 3.3
|
205
|
+
rubygems_version: 3.4.3
|
187
206
|
signing_key:
|
188
207
|
specification_version: 4
|
189
208
|
summary: Sidekiq strategy to support queue limits
|
190
|
-
test_files:
|
191
|
-
- spec/sidekiq/extensions/queue_spec.rb
|
192
|
-
- spec/sidekiq/limit_fetch/global/monitor_spec.rb
|
193
|
-
- spec/sidekiq/limit_fetch/queues_spec.rb
|
194
|
-
- spec/sidekiq/limit_fetch/semaphore_spec.rb
|
195
|
-
- spec/sidekiq/limit_fetch_spec.rb
|
196
|
-
- spec/spec_helper.rb
|
209
|
+
test_files: []
|