sidekiq-limit_fetch 4.3.2 → 4.4.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (48) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci.yml +3 -1
  3. data/.rubocop.yml +29 -0
  4. data/Appraisals +6 -0
  5. data/Gemfile +2 -0
  6. data/README.md +11 -1
  7. data/Rakefile +2 -0
  8. data/bench/compare.rb +17 -13
  9. data/demo/Gemfile +3 -2
  10. data/demo/Rakefile +6 -5
  11. data/demo/app/workers/a_worker.rb +2 -0
  12. data/demo/app/workers/b_worker.rb +2 -0
  13. data/demo/app/workers/c_worker.rb +2 -1
  14. data/demo/app/workers/fast_worker.rb +2 -0
  15. data/demo/app/workers/slow_worker.rb +2 -0
  16. data/demo/config/application.rb +3 -1
  17. data/demo/config/boot.rb +4 -2
  18. data/demo/config/environment.rb +3 -1
  19. data/demo/config/environments/development.rb +2 -0
  20. data/docker-compose.dev.yml +13 -0
  21. data/gemfiles/sidekiq_6.0.gemfile.lock +3 -4
  22. data/gemfiles/sidekiq_6.1.gemfile.lock +3 -4
  23. data/gemfiles/sidekiq_6.2.gemfile.lock +3 -4
  24. data/gemfiles/sidekiq_6.3.gemfile.lock +3 -4
  25. data/gemfiles/sidekiq_6.4.gemfile.lock +3 -4
  26. data/gemfiles/sidekiq_6.5.gemfile.lock +3 -4
  27. data/gemfiles/sidekiq_7.0.gemfile +7 -0
  28. data/gemfiles/sidekiq_7.0.gemfile.lock +58 -0
  29. data/gemfiles/sidekiq_master.gemfile.lock +3 -4
  30. data/lib/sidekiq/extensions/manager.rb +21 -13
  31. data/lib/sidekiq/extensions/queue.rb +16 -13
  32. data/lib/sidekiq/limit_fetch/global/monitor.rb +64 -53
  33. data/lib/sidekiq/limit_fetch/global/selector.rb +49 -40
  34. data/lib/sidekiq/limit_fetch/global/semaphore.rb +130 -123
  35. data/lib/sidekiq/limit_fetch/instances.rb +22 -16
  36. data/lib/sidekiq/limit_fetch/queues.rb +165 -124
  37. data/lib/sidekiq/limit_fetch/unit_of_work.rb +26 -22
  38. data/lib/sidekiq/limit_fetch.rb +73 -54
  39. data/lib/sidekiq-limit_fetch.rb +2 -0
  40. data/sidekiq-limit_fetch.gemspec +20 -13
  41. data/spec/sidekiq/extensions/manager_spec.rb +19 -0
  42. data/spec/sidekiq/extensions/queue_spec.rb +2 -0
  43. data/spec/sidekiq/limit_fetch/global/monitor_spec.rb +86 -5
  44. data/spec/sidekiq/limit_fetch/queues_spec.rb +34 -18
  45. data/spec/sidekiq/limit_fetch/semaphore_spec.rb +2 -0
  46. data/spec/sidekiq/limit_fetch_spec.rb +14 -4
  47. data/spec/spec_helper.rb +15 -4
  48. metadata +34 -21
@@ -1,22 +1,24 @@
1
+ # frozen_string_literal: true
2
+
1
3
  RSpec.describe Sidekiq::LimitFetch::Global::Monitor do
2
4
  let(:monitor) { described_class.start! ttl, timeout }
3
5
  let(:ttl) { 1 }
4
6
  let(:queue) { Sidekiq::Queue[name] }
5
7
  let(:name) { 'default' }
8
+ let(:timeout) { 0.5 }
6
9
 
7
- before { monitor }
8
10
  after { monitor.kill }
9
11
 
10
12
  context 'old locks' do
11
- let(:timeout) { 0.5 }
13
+ before { monitor }
12
14
 
13
15
  it 'should remove invalidated old locks' do
14
16
  2.times { queue.acquire }
15
- sleep 2*ttl
17
+ sleep ttl * 2
16
18
  expect(queue.probed).to eq 2
17
19
 
18
20
  allow(described_class).to receive(:update_heartbeat)
19
- sleep 2*ttl
21
+ sleep ttl * 2
20
22
  expect(queue.probed).to eq 0
21
23
  end
22
24
 
@@ -26,8 +28,87 @@ RSpec.describe Sidekiq::LimitFetch::Global::Monitor do
26
28
  Sidekiq.redis do |it|
27
29
  it.del Sidekiq::LimitFetch::Global::Monitor::PROCESS_SET
28
30
  end
29
- sleep 2*ttl
31
+ sleep ttl * 2
30
32
  expect(queue.probed).to eq 0
31
33
  end
32
34
  end
35
+
36
+ context 'dynamic queue' do
37
+ let(:limits) do
38
+ {
39
+ 'queue1' => 3,
40
+ 'queue2' => 3
41
+ }
42
+ end
43
+ let(:queues) { %w[queue1 queue2] }
44
+ let(:queue) { Sidekiq::LimitFetch::Queues }
45
+
46
+ let(:config) { Sidekiq::Config.new(options) }
47
+ let(:capsule) do
48
+ config.capsule('default') do |cap|
49
+ cap.concurrency = 1
50
+ cap.queues = config[:queues]
51
+ end
52
+ end
53
+
54
+ let(:capsule_or_options) do
55
+ Sidekiq::LimitFetch.post_7? ? capsule : options
56
+ end
57
+
58
+ context 'without excluded queue' do
59
+ let(:options) do
60
+ {
61
+ limits: limits,
62
+ queues: queues,
63
+ dynamic: true
64
+ }
65
+ end
66
+
67
+ it 'should add dynamic queue' do
68
+ queue.start(capsule_or_options)
69
+ monitor
70
+
71
+ expect(queue.instance_variable_get(:@queues)).not_to include('queue3')
72
+
73
+ Sidekiq.redis do |it|
74
+ it.sadd 'queues', 'queue3'
75
+ end
76
+
77
+ sleep ttl * 2
78
+ expect(queue.instance_variable_get(:@queues)).to include('queue3')
79
+
80
+ Sidekiq.redis do |it|
81
+ it.srem 'queues', 'queue3'
82
+ end
83
+ end
84
+ end
85
+
86
+ context 'with excluded queue' do
87
+ let(:options) do
88
+ {
89
+ limits: limits,
90
+ queues: queues,
91
+ dynamic: { exclude: ['queue4'] }
92
+ }
93
+ end
94
+
95
+ it 'should exclude excluded dynamic queue' do
96
+ queue.start(capsule_or_options)
97
+ monitor
98
+
99
+ expect(queue.instance_variable_get(:@queues)).not_to include('queue4')
100
+
101
+ Sidekiq.redis do |it|
102
+ it.sadd 'queues', 'queue4'
103
+ end
104
+
105
+ sleep ttl * 2
106
+ expect(queue.instance_variable_get(:@queues)).not_to include('queue4')
107
+
108
+ Sidekiq.redis do |it|
109
+ it.srem 'queues', 'queue4'
110
+ end
111
+ end
112
+ end
113
+ end
33
114
  end
@@ -1,19 +1,35 @@
1
+ # frozen_string_literal: true
2
+
1
3
  RSpec.describe Sidekiq::LimitFetch::Queues do
2
4
  let(:queues) { %w[queue1 queue2] }
3
- let(:limits) {{ 'queue1' => 3 }}
5
+ let(:limits) { { 'queue1' => 3 } }
4
6
  let(:strict) { true }
5
- let(:blocking) {}
6
- let(:process_limits) {{ 'queue2' => 3 }}
7
+ let(:blocking) { nil }
8
+ let(:process_limits) { { 'queue2' => 3 } }
7
9
 
8
10
  let(:options) do
9
- { queues: queues,
10
- limits: limits,
11
- strict: strict,
11
+ { queues: queues,
12
+ limits: limits,
13
+ strict: strict,
12
14
  blocking: blocking,
13
15
  process_limits: process_limits }
14
16
  end
15
17
 
16
- before { subject.start options }
18
+ let(:config) { Sidekiq::Config.new(options) }
19
+ let(:capsule) do
20
+ config.capsule('default') do |cap|
21
+ cap.concurrency = 1
22
+ cap.queues = config[:queues]
23
+ end
24
+ end
25
+
26
+ let(:capsule_or_options) do
27
+ Sidekiq::LimitFetch.post_7? ? capsule : options
28
+ end
29
+
30
+ before do
31
+ subject.start(capsule_or_options)
32
+ end
17
33
 
18
34
  def in_thread(&block)
19
35
  thr = Thread.new(&block)
@@ -51,36 +67,36 @@ RSpec.describe Sidekiq::LimitFetch::Queues do
51
67
  end
52
68
 
53
69
  it 'should release queues' do
54
- in_thread {
70
+ in_thread do
55
71
  subject.acquire
56
72
  subject.release_except nil
57
- }
73
+ end
58
74
  expect(Sidekiq::Queue['queue1'].probed).to eq 0
59
75
  expect(Sidekiq::Queue['queue2'].probed).to eq 0
60
76
  end
61
77
 
62
78
  it 'should release queues except selected' do
63
- in_thread {
79
+ in_thread do
64
80
  subject.acquire
65
81
  subject.release_except 'queue:queue1'
66
- }
82
+ end
67
83
  expect(Sidekiq::Queue['queue1'].probed).to eq 1
68
84
  expect(Sidekiq::Queue['queue2'].probed).to eq 0
69
85
  end
70
86
 
71
87
  it 'should release when no queues was acquired' do
72
- queues.each {|name| Sidekiq::Queue[name].pause }
73
- in_thread {
88
+ queues.each { |name| Sidekiq::Queue[name].pause }
89
+ in_thread do
74
90
  subject.acquire
75
91
  expect { subject.release_except nil }.not_to raise_exception
76
- }
92
+ end
77
93
  end
78
94
 
79
95
  context 'blocking' do
80
- let(:blocking) { %w(queue1) }
96
+ let(:blocking) { %w[queue1] }
81
97
 
82
98
  it 'should acquire blocking queues' do
83
- 3.times { in_thread { subject.acquire } }
99
+ 3.times { in_thread { subject.acquire } }
84
100
  expect(Sidekiq::Queue['queue1'].probed).to eq 3
85
101
  expect(Sidekiq::Queue['queue2'].probed).to eq 1
86
102
  end
@@ -101,11 +117,11 @@ RSpec.describe Sidekiq::LimitFetch::Queues do
101
117
  let(:strict) { false }
102
118
 
103
119
  it 'should retrieve weighted queues' do
104
- expect(subject.ordered_queues).to match_array(%w(queue1 queue2))
120
+ expect(subject.ordered_queues).to match_array(%w[queue1 queue2])
105
121
  end
106
122
  end
107
123
 
108
124
  it 'with strict flag should retrieve strictly ordered queues' do
109
- expect(subject.ordered_queues).to eq %w(queue1 queue2)
125
+ expect(subject.ordered_queues).to eq %w[queue1 queue2]
110
126
  end
111
127
  end
@@ -1,3 +1,5 @@
1
+ # frozen_string_literal: true
2
+
1
3
  RSpec.describe 'semaphore' do
2
4
  let(:name) { 'default' }
3
5
  subject { Sidekiq::LimitFetch::Global::Semaphore.new name }
@@ -1,12 +1,22 @@
1
+ # frozen_string_literal: true
2
+
1
3
  Thread.abort_on_exception = true
2
4
 
3
5
  RSpec.describe Sidekiq::LimitFetch do
4
- let(:options) {{ queues: queues, limits: limits }}
5
- let(:queues) { %w(queue1 queue1 queue2 queue2) }
6
- let(:limits) {{ 'queue1' => 1, 'queue2' => 2 }}
6
+ let(:options) { { queues: queues, limits: limits } }
7
+ let(:queues) { %w[queue1 queue1 queue2 queue2] }
8
+ let(:limits) { { 'queue1' => 1, 'queue2' => 2 } }
9
+ let(:config) { Sidekiq::Config.new(options) }
10
+ let(:capsule) do
11
+ config.capsule('default') do |cap|
12
+ cap.concurrency = 1
13
+ cap.queues = config[:queues]
14
+ end
15
+ end
16
+ let(:capsule_or_config) { Sidekiq::LimitFetch.post_7? ? capsule : options }
7
17
 
8
18
  before do
9
- subject::Queues.start options
19
+ subject::Queues.start(capsule_or_config)
10
20
 
11
21
  Sidekiq.redis do |it|
12
22
  it.del 'queue:queue1'
data/spec/spec_helper.rb CHANGED
@@ -1,7 +1,18 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'simplecov'
4
+ SimpleCov.start
5
+
1
6
  require 'sidekiq/limit_fetch'
2
7
 
3
- Sidekiq.logger = nil
4
- Sidekiq.redis = { namespace: ENV['namespace'] }
8
+ if Sidekiq::LimitFetch.post_7?
9
+ Sidekiq.configure_embed do |config|
10
+ config.logger = nil
11
+ end
12
+ else
13
+ Sidekiq.logger = nil
14
+ Sidekiq.redis = { namespace: ENV.fetch('namespace', nil) }
15
+ end
5
16
 
6
17
  RSpec.configure do |config|
7
18
  config.order = :random
@@ -10,7 +21,7 @@ RSpec.configure do |config|
10
21
  config.before do
11
22
  Sidekiq::Queue.reset_instances!
12
23
  Sidekiq.redis do |it|
13
- clean_redis = ->(queue) do
24
+ clean_redis = lambda do |queue|
14
25
  it.pipelined do |pipeline|
15
26
  pipeline.del "limit_fetch:limit:#{queue}"
16
27
  pipeline.del "limit_fetch:process_limit:#{queue}"
@@ -22,7 +33,7 @@ RSpec.configure do |config|
22
33
  end
23
34
 
24
35
  clean_redis.call(name) if defined?(name)
25
- queues.each(&clean_redis) if defined?(queues) and queues.is_a? Array
36
+ queues.each(&clean_redis) if defined?(queues) && queues.is_a?(Array)
26
37
  end
27
38
  end
28
39
  end
metadata CHANGED
@@ -1,15 +1,14 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: sidekiq-limit_fetch
3
3
  version: !ruby/object:Gem::Version
4
- version: 4.3.2
4
+ version: 4.4.1
5
5
  platform: ruby
6
6
  authors:
7
7
  - Dean Perry
8
- - brainopia
9
8
  autorequire:
10
9
  bindir: bin
11
10
  cert_chain: []
12
- date: 2022-09-01 00:00:00.000000000 Z
11
+ date: 2023-01-22 00:00:00.000000000 Z
13
12
  dependencies:
14
13
  - !ruby/object:Gem::Dependency
15
14
  name: sidekiq
@@ -17,28 +16,42 @@ dependencies:
17
16
  requirements:
18
17
  - - ">="
19
18
  - !ruby/object:Gem::Version
20
- version: '4'
19
+ version: '6'
21
20
  type: :runtime
22
21
  prerelease: false
23
22
  version_requirements: !ruby/object:Gem::Requirement
24
23
  requirements:
25
24
  - - ">="
26
25
  - !ruby/object:Gem::Version
27
- version: '4'
26
+ version: '6'
28
27
  - !ruby/object:Gem::Dependency
29
- name: redis
28
+ name: appraisal
30
29
  requirement: !ruby/object:Gem::Requirement
31
30
  requirements:
32
31
  - - ">="
33
32
  - !ruby/object:Gem::Version
34
- version: 4.6.0
35
- type: :runtime
33
+ version: '0'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - ">="
39
+ - !ruby/object:Gem::Version
40
+ version: '0'
41
+ - !ruby/object:Gem::Dependency
42
+ name: rake
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - ">="
46
+ - !ruby/object:Gem::Version
47
+ version: '0'
48
+ type: :development
36
49
  prerelease: false
37
50
  version_requirements: !ruby/object:Gem::Requirement
38
51
  requirements:
39
52
  - - ">="
40
53
  - !ruby/object:Gem::Version
41
- version: 4.6.0
54
+ version: '0'
42
55
  - !ruby/object:Gem::Dependency
43
56
  name: redis-namespace
44
57
  requirement: !ruby/object:Gem::Requirement
@@ -60,7 +73,7 @@ dependencies:
60
73
  - !ruby/object:Gem::Version
61
74
  version: 1.5.2
62
75
  - !ruby/object:Gem::Dependency
63
- name: appraisal
76
+ name: rspec
64
77
  requirement: !ruby/object:Gem::Requirement
65
78
  requirements:
66
79
  - - ">="
@@ -74,7 +87,7 @@ dependencies:
74
87
  - !ruby/object:Gem::Version
75
88
  version: '0'
76
89
  - !ruby/object:Gem::Dependency
77
- name: rspec
90
+ name: rubocop
78
91
  requirement: !ruby/object:Gem::Requirement
79
92
  requirements:
80
93
  - - ">="
@@ -88,7 +101,7 @@ dependencies:
88
101
  - !ruby/object:Gem::Version
89
102
  version: '0'
90
103
  - !ruby/object:Gem::Dependency
91
- name: rake
104
+ name: simplecov
92
105
  requirement: !ruby/object:Gem::Requirement
93
106
  requirements:
94
107
  - - ">="
@@ -111,6 +124,7 @@ files:
111
124
  - ".github/workflows/ci.yml"
112
125
  - ".gitignore"
113
126
  - ".rspec"
127
+ - ".rubocop.yml"
114
128
  - Appraisals
115
129
  - CHANGELOG.md
116
130
  - Gemfile
@@ -130,6 +144,7 @@ files:
130
144
  - demo/config/boot.rb
131
145
  - demo/config/environment.rb
132
146
  - demo/config/environments/development.rb
147
+ - docker-compose.dev.yml
133
148
  - gemfiles/sidekiq_6.0.gemfile
134
149
  - gemfiles/sidekiq_6.0.gemfile.lock
135
150
  - gemfiles/sidekiq_6.1.gemfile
@@ -142,6 +157,8 @@ files:
142
157
  - gemfiles/sidekiq_6.4.gemfile.lock
143
158
  - gemfiles/sidekiq_6.5.gemfile
144
159
  - gemfiles/sidekiq_6.5.gemfile.lock
160
+ - gemfiles/sidekiq_7.0.gemfile
161
+ - gemfiles/sidekiq_7.0.gemfile.lock
145
162
  - gemfiles/sidekiq_master.gemfile
146
163
  - gemfiles/sidekiq_master.gemfile.lock
147
164
  - lib/sidekiq-limit_fetch.rb
@@ -155,6 +172,7 @@ files:
155
172
  - lib/sidekiq/limit_fetch/queues.rb
156
173
  - lib/sidekiq/limit_fetch/unit_of_work.rb
157
174
  - sidekiq-limit_fetch.gemspec
175
+ - spec/sidekiq/extensions/manager_spec.rb
158
176
  - spec/sidekiq/extensions/queue_spec.rb
159
177
  - spec/sidekiq/limit_fetch/global/monitor_spec.rb
160
178
  - spec/sidekiq/limit_fetch/queues_spec.rb
@@ -168,6 +186,7 @@ metadata:
168
186
  homepage_uri: https://github.com/deanpcmad/sidekiq-limit_fetch
169
187
  source_code_uri: https://github.com/deanpcmad/sidekiq-limit_fetch
170
188
  changelog_uri: https://github.com/deanpcmad/sidekiq-limit_fetch/blob/master/CHANGELOG.md
189
+ rubygems_mfa_required: 'true'
171
190
  post_install_message:
172
191
  rdoc_options: []
173
192
  require_paths:
@@ -176,21 +195,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
176
195
  requirements:
177
196
  - - ">="
178
197
  - !ruby/object:Gem::Version
179
- version: '0'
198
+ version: 2.7.0
180
199
  required_rubygems_version: !ruby/object:Gem::Requirement
181
200
  requirements:
182
201
  - - ">="
183
202
  - !ruby/object:Gem::Version
184
203
  version: '0'
185
204
  requirements: []
186
- rubygems_version: 3.3.7
205
+ rubygems_version: 3.4.3
187
206
  signing_key:
188
207
  specification_version: 4
189
208
  summary: Sidekiq strategy to support queue limits
190
- test_files:
191
- - spec/sidekiq/extensions/queue_spec.rb
192
- - spec/sidekiq/limit_fetch/global/monitor_spec.rb
193
- - spec/sidekiq/limit_fetch/queues_spec.rb
194
- - spec/sidekiq/limit_fetch/semaphore_spec.rb
195
- - spec/sidekiq/limit_fetch_spec.rb
196
- - spec/spec_helper.rb
209
+ test_files: []