sidekiq-limit_fetch 4.4.0 → 4.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (38) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci.yml +2 -0
  3. data/.rubocop.yml +29 -0
  4. data/Appraisals +2 -0
  5. data/Gemfile +2 -0
  6. data/Rakefile +2 -0
  7. data/bench/compare.rb +17 -13
  8. data/demo/Gemfile +3 -2
  9. data/demo/Rakefile +6 -5
  10. data/demo/app/workers/a_worker.rb +2 -0
  11. data/demo/app/workers/b_worker.rb +2 -0
  12. data/demo/app/workers/c_worker.rb +2 -1
  13. data/demo/app/workers/fast_worker.rb +2 -0
  14. data/demo/app/workers/slow_worker.rb +2 -0
  15. data/demo/config/application.rb +3 -1
  16. data/demo/config/boot.rb +4 -2
  17. data/demo/config/environment.rb +3 -1
  18. data/demo/config/environments/development.rb +2 -0
  19. data/docker-compose.dev.yml +2 -2
  20. data/lib/sidekiq/extensions/manager.rb +20 -17
  21. data/lib/sidekiq/extensions/queue.rb +16 -13
  22. data/lib/sidekiq/limit_fetch/global/monitor.rb +64 -58
  23. data/lib/sidekiq/limit_fetch/global/selector.rb +49 -44
  24. data/lib/sidekiq/limit_fetch/global/semaphore.rb +130 -123
  25. data/lib/sidekiq/limit_fetch/instances.rb +22 -16
  26. data/lib/sidekiq/limit_fetch/queues.rb +163 -137
  27. data/lib/sidekiq/limit_fetch/unit_of_work.rb +26 -22
  28. data/lib/sidekiq/limit_fetch.rb +68 -64
  29. data/lib/sidekiq-limit_fetch.rb +2 -0
  30. data/sidekiq-limit_fetch.gemspec +18 -10
  31. data/spec/sidekiq/extensions/manager_spec.rb +19 -0
  32. data/spec/sidekiq/extensions/queue_spec.rb +2 -0
  33. data/spec/sidekiq/limit_fetch/global/monitor_spec.rb +59 -32
  34. data/spec/sidekiq/limit_fetch/queues_spec.rb +34 -18
  35. data/spec/sidekiq/limit_fetch/semaphore_spec.rb +2 -0
  36. data/spec/sidekiq/limit_fetch_spec.rb +14 -4
  37. data/spec/spec_helper.rb +8 -3
  38. metadata +39 -14
@@ -1,136 +1,141 @@
1
- module Sidekiq::LimitFetch::Global
2
- class Semaphore
3
- PREFIX = 'limit_fetch'
1
+ # frozen_string_literal: true
4
2
 
5
- attr_reader :local_busy
3
+ module Sidekiq
4
+ module LimitFetch
5
+ module Global
6
+ class Semaphore
7
+ PREFIX = 'limit_fetch'
6
8
 
7
- def initialize(name)
8
- @name = name
9
- @lock = Mutex.new
10
- @local_busy = 0
11
- end
9
+ attr_reader :local_busy
12
10
 
13
- def limit
14
- value = redis {|it| it.get "#{PREFIX}:limit:#@name" }
15
- value.to_i if value
16
- end
11
+ def initialize(name)
12
+ @name = name
13
+ @lock = Mutex.new
14
+ @local_busy = 0
15
+ end
17
16
 
18
- def limit=(value)
19
- @limit_changed = true
17
+ def limit
18
+ value = redis { |it| it.get "#{PREFIX}:limit:#{@name}" }
19
+ value&.to_i
20
+ end
20
21
 
21
- if value
22
- redis {|it| it.set "#{PREFIX}:limit:#@name", value }
23
- else
24
- redis {|it| it.del "#{PREFIX}:limit:#@name" }
25
- end
26
- end
22
+ def limit=(value)
23
+ @limit_changed = true
27
24
 
28
- def limit_changed?
29
- @limit_changed
30
- end
25
+ if value
26
+ redis { |it| it.set "#{PREFIX}:limit:#{@name}", value }
27
+ else
28
+ redis { |it| it.del "#{PREFIX}:limit:#{@name}" }
29
+ end
30
+ end
31
31
 
32
- def process_limit
33
- value = redis {|it| it.get "#{PREFIX}:process_limit:#@name" }
34
- value.to_i if value
35
- end
32
+ def limit_changed?
33
+ @limit_changed
34
+ end
36
35
 
37
- def process_limit=(value)
38
- if value
39
- redis {|it| it.set "#{PREFIX}:process_limit:#@name", value }
40
- else
41
- redis {|it| it.del "#{PREFIX}:process_limit:#@name" }
42
- end
43
- end
36
+ def process_limit
37
+ value = redis { |it| it.get "#{PREFIX}:process_limit:#{@name}" }
38
+ value&.to_i
39
+ end
44
40
 
45
- def acquire
46
- Selector.acquire([@name], namespace).size > 0
47
- end
41
+ def process_limit=(value)
42
+ if value
43
+ redis { |it| it.set "#{PREFIX}:process_limit:#{@name}", value }
44
+ else
45
+ redis { |it| it.del "#{PREFIX}:process_limit:#{@name}" }
46
+ end
47
+ end
48
48
 
49
- def release
50
- redis {|it| it.lrem "#{PREFIX}:probed:#@name", 1, Selector.uuid }
51
- end
49
+ def acquire
50
+ Selector.acquire([@name], namespace).size.positive?
51
+ end
52
52
 
53
- def busy
54
- redis {|it| it.llen "#{PREFIX}:busy:#@name" }
55
- end
53
+ def release
54
+ redis { |it| it.lrem "#{PREFIX}:probed:#{@name}", 1, Selector.uuid }
55
+ end
56
56
 
57
- def busy_processes
58
- redis {|it| it.lrange "#{PREFIX}:busy:#@name", 0, -1 }
59
- end
57
+ def busy
58
+ redis { |it| it.llen "#{PREFIX}:busy:#{@name}" }
59
+ end
60
60
 
61
- def increase_busy
62
- increase_local_busy
63
- redis {|it| it.rpush "#{PREFIX}:busy:#@name", Selector.uuid }
64
- end
61
+ def busy_processes
62
+ redis { |it| it.lrange "#{PREFIX}:busy:#{@name}", 0, -1 }
63
+ end
65
64
 
66
- def decrease_busy
67
- decrease_local_busy
68
- redis {|it| it.lrem "#{PREFIX}:busy:#@name", 1, Selector.uuid }
69
- end
65
+ def increase_busy
66
+ increase_local_busy
67
+ redis { |it| it.rpush "#{PREFIX}:busy:#{@name}", Selector.uuid }
68
+ end
70
69
 
71
- def probed
72
- redis {|it| it.llen "#{PREFIX}:probed:#@name" }
73
- end
70
+ def decrease_busy
71
+ decrease_local_busy
72
+ redis { |it| it.lrem "#{PREFIX}:busy:#{@name}", 1, Selector.uuid }
73
+ end
74
74
 
75
- def probed_processes
76
- redis {|it| it.lrange "#{PREFIX}:probed:#@name", 0, -1 }
77
- end
75
+ def probed
76
+ redis { |it| it.llen "#{PREFIX}:probed:#{@name}" }
77
+ end
78
78
 
79
- def pause
80
- redis {|it| it.set "#{PREFIX}:pause:#@name", '1' }
81
- end
79
+ def probed_processes
80
+ redis { |it| it.lrange "#{PREFIX}:probed:#{@name}", 0, -1 }
81
+ end
82
82
 
83
- def pause_for_ms ms
84
- redis {|it| it.psetex "#{PREFIX}:pause:#@name", ms, 1 }
85
- end
83
+ def pause
84
+ redis { |it| it.set "#{PREFIX}:pause:#{@name}", '1' }
85
+ end
86
86
 
87
- def unpause
88
- redis {|it| it.del "#{PREFIX}:pause:#@name" }
89
- end
87
+ def pause_for_ms(milliseconds)
88
+ redis { |it| it.psetex "#{PREFIX}:pause:#{@name}", milliseconds, 1 }
89
+ end
90
90
 
91
- def paused?
92
- redis {|it| it.get "#{PREFIX}:pause:#@name" } == '1'
93
- end
91
+ def unpause
92
+ redis { |it| it.del "#{PREFIX}:pause:#{@name}" }
93
+ end
94
94
 
95
- def block
96
- redis {|it| it.set "#{PREFIX}:block:#@name", '1' }
97
- end
95
+ def paused?
96
+ redis { |it| it.get "#{PREFIX}:pause:#{@name}" } == '1'
97
+ end
98
98
 
99
- def block_except(*queues)
100
- raise ArgumentError if queues.empty?
101
- redis {|it| it.set "#{PREFIX}:block:#@name", queues.join(',') }
102
- end
99
+ def block
100
+ redis { |it| it.set "#{PREFIX}:block:#{@name}", '1' }
101
+ end
103
102
 
104
- def unblock
105
- redis {|it| it.del "#{PREFIX}:block:#@name" }
106
- end
103
+ def block_except(*queues)
104
+ raise ArgumentError if queues.empty?
107
105
 
108
- def blocking?
109
- redis {|it| it.get "#{PREFIX}:block:#@name" } == '1'
110
- end
106
+ redis { |it| it.set "#{PREFIX}:block:#{@name}", queues.join(',') }
107
+ end
111
108
 
112
- def clear_limits
113
- redis do |it|
114
- %w(block busy limit pause probed process_limit).each do |key|
115
- it.del "#{PREFIX}:#{key}:#@name"
109
+ def unblock
110
+ redis { |it| it.del "#{PREFIX}:block:#{@name}" }
116
111
  end
117
- end
118
- end
119
112
 
120
- def increase_local_busy
121
- @lock.synchronize { @local_busy += 1 }
122
- end
113
+ def blocking?
114
+ redis { |it| it.get "#{PREFIX}:block:#{@name}" } == '1'
115
+ end
123
116
 
124
- def decrease_local_busy
125
- @lock.synchronize { @local_busy -= 1 }
126
- end
117
+ def clear_limits
118
+ redis do |it|
119
+ %w[block busy limit pause probed process_limit].each do |key|
120
+ it.del "#{PREFIX}:#{key}:#{@name}"
121
+ end
122
+ end
123
+ end
127
124
 
128
- def local_busy?
129
- @local_busy > 0
130
- end
125
+ def increase_local_busy
126
+ @lock.synchronize { @local_busy += 1 }
127
+ end
128
+
129
+ def decrease_local_busy
130
+ @lock.synchronize { @local_busy -= 1 }
131
+ end
132
+
133
+ def local_busy?
134
+ @local_busy.positive?
135
+ end
131
136
 
132
- def explain
133
- <<-END.gsub(/^ {8}/, '')
137
+ def explain
138
+ <<-INFO.gsub(/^ {8}/, '')
134
139
  Current sidekiq process: #{Selector.uuid}
135
140
 
136
141
  All processes:
@@ -153,31 +158,33 @@ module Sidekiq::LimitFetch::Global
153
158
 
154
159
  Blocking:
155
160
  #{blocking?}
156
- END
157
- end
161
+ INFO
162
+ end
158
163
 
159
- def remove_locks_except!(processes)
160
- locked_processes = probed_processes.uniq
161
- (locked_processes - processes).each do |dead_process|
162
- remove_lock! dead_process
163
- end
164
- end
164
+ def remove_locks_except!(processes)
165
+ locked_processes = probed_processes.uniq
166
+ (locked_processes - processes).each do |dead_process|
167
+ remove_lock! dead_process
168
+ end
169
+ end
165
170
 
166
- def remove_lock!(process)
167
- redis do |it|
168
- it.lrem "#{PREFIX}:probed:#@name", 0, process
169
- it.lrem "#{PREFIX}:busy:#@name", 0, process
170
- end
171
- end
171
+ def remove_lock!(process)
172
+ redis do |it|
173
+ it.lrem "#{PREFIX}:probed:#{@name}", 0, process
174
+ it.lrem "#{PREFIX}:busy:#{@name}", 0, process
175
+ end
176
+ end
172
177
 
173
- private
178
+ private
174
179
 
175
- def redis(&block)
176
- Sidekiq.redis(&block)
177
- end
180
+ def redis(&block)
181
+ Sidekiq.redis(&block)
182
+ end
178
183
 
179
- def namespace
180
- Sidekiq::LimitFetch::Queues.namespace
184
+ def namespace
185
+ Sidekiq::LimitFetch::Queues.namespace
186
+ end
187
+ end
181
188
  end
182
189
  end
183
190
  end
@@ -1,23 +1,29 @@
1
- module Sidekiq::LimitFetch::Instances
2
- def self.extended(klass)
3
- klass.instance_variable_set :@instances, {}
4
- end
1
+ # frozen_string_literal: true
5
2
 
6
- def new(*args)
7
- @instances[args] ||= super
8
- end
3
+ module Sidekiq
4
+ module LimitFetch
5
+ module Instances
6
+ def self.extended(klass)
7
+ klass.instance_variable_set :@instances, {}
8
+ end
9
9
 
10
- alias [] new
10
+ def new(*args)
11
+ @instances[args] ||= super
12
+ end
11
13
 
12
- def instances
13
- @instances.values
14
- end
14
+ alias [] new
15
15
 
16
- def reset_instances!
17
- @instances = {}
18
- end
16
+ def instances
17
+ @instances.values
18
+ end
19
+
20
+ def reset_instances!
21
+ @instances = {}
22
+ end
19
23
 
20
- def delete_instance(name)
21
- @instances.delete [name]
24
+ def delete_instance(name)
25
+ @instances.delete [name]
26
+ end
27
+ end
22
28
  end
23
29
  end
@@ -1,174 +1,200 @@
1
- module Sidekiq::LimitFetch::Queues
2
- extend self
1
+ # frozen_string_literal: true
2
+
3
+ module Sidekiq
4
+ module LimitFetch
5
+ module Queues
6
+ extend self
7
+
8
+ THREAD_KEY = :acquired_queues
9
+
10
+ # rubocop:disable Metrics/AbcSize
11
+ # rubocop:disable Metrics/CyclomaticComplexity
12
+ # rubocop:disable Metrics/MethodLength
13
+ # rubocop:disable Metrics/PerceivedComplexity
14
+ def start(capsule_or_options)
15
+ config = Sidekiq::LimitFetch.post_7? ? capsule_or_options.config : capsule_or_options
16
+
17
+ @queues = config[:queues].map do |queue|
18
+ if queue.is_a? Array
19
+ queue.first
20
+ else
21
+ queue
22
+ end
23
+ end.uniq
24
+ @startup_queues = @queues.dup
25
+
26
+ if config[:dynamic].is_a? Hash
27
+ @dynamic = true
28
+ @dynamic_exclude = config[:dynamic][:exclude] || []
29
+ else
30
+ @dynamic = config[:dynamic]
31
+ @dynamic_exclude = []
32
+ end
3
33
 
4
- THREAD_KEY = :acquired_queues
34
+ @limits = config[:limits] || {}
35
+ @process_limits = config[:process_limits] || {}
36
+ @blocks = config[:blocking] || []
5
37
 
6
- def start(options)
7
- if Sidekiq::LimitFetch.post_7?
8
- options = options.config
9
- end
38
+ config[:strict] ? strict_order! : weighted_order!
10
39
 
11
- @queues = options[:queues]
12
- @startup_queues = options[:queues].dup
40
+ apply_process_limit_to_queues
41
+ apply_limit_to_queues
42
+ apply_blocks_to_queues
43
+ end
44
+ # rubocop:enable Metrics/AbcSize
45
+ # rubocop:enable Metrics/CyclomaticComplexity
46
+ # rubocop:enable Metrics/MethodLength
47
+ # rubocop:enable Metrics/PerceivedComplexity
48
+
49
+ def acquire
50
+ queues = saved
51
+ queues ||= Sidekiq::LimitFetch.redis_retryable do
52
+ selector.acquire(ordered_queues, namespace)
53
+ end
54
+ save queues
55
+ queues.map { |it| "queue:#{it}" }
56
+ end
13
57
 
14
- if options[:dynamic].is_a? Hash
15
- @dynamic = true
16
- @dynamic_exclude = options[:dynamic][:exclude] || []
17
- else
18
- @dynamic = options[:dynamic]
19
- @dynamic_exclude = []
20
- end
58
+ def release_except(full_name)
59
+ queues = restore
60
+ queues.delete full_name[/queue:(.*)/, 1] if full_name
61
+ Sidekiq::LimitFetch.redis_retryable do
62
+ selector.release queues, namespace
63
+ end
64
+ end
21
65
 
22
- @limits = options[:limits] || {}
23
- @process_limits = options[:process_limits] || {}
24
- @blocks = options[:blocking] || []
66
+ def dynamic?
67
+ @dynamic
68
+ end
25
69
 
26
- options[:strict] ? strict_order! : weighted_order!
70
+ def startup_queue?(queue)
71
+ @startup_queues.include?(queue)
72
+ end
27
73
 
28
- apply_process_limit_to_queues
29
- apply_limit_to_queues
30
- apply_blocks_to_queues
31
- end
74
+ def dynamic_exclude
75
+ @dynamic_exclude
76
+ end
32
77
 
33
- def acquire
34
- queues = saved
35
- queues ||= Sidekiq::LimitFetch.redis_retryable do
36
- selector.acquire(ordered_queues, namespace)
37
- end
38
- save queues
39
- queues.map { |it| "queue:#{it}" }
40
- end
78
+ def add(queues)
79
+ return unless queues
41
80
 
42
- def release_except(full_name)
43
- queues = restore
44
- queues.delete full_name[/queue:(.*)/, 1] if full_name
45
- Sidekiq::LimitFetch.redis_retryable do
46
- selector.release queues, namespace
47
- end
48
- end
81
+ queues.each do |queue|
82
+ next if @queues.include? queue
49
83
 
50
- def dynamic?
51
- @dynamic
52
- end
84
+ if startup_queue?(queue)
85
+ apply_process_limit_to_queue(queue)
86
+ apply_limit_to_queue(queue)
87
+ end
53
88
 
54
- def startup_queue?(queue)
55
- @startup_queues.include?(queue)
56
- end
89
+ @queues.push queue
90
+ end
91
+ end
57
92
 
58
- def dynamic_exclude
59
- @dynamic_exclude
60
- end
93
+ def remove(queues)
94
+ return unless queues
61
95
 
62
- def add(queues)
63
- return unless queues
64
- queues.each do |queue|
65
- unless @queues.include? queue
66
- if startup_queue?(queue)
67
- apply_process_limit_to_queue(queue)
68
- apply_limit_to_queue(queue)
69
- end
96
+ queues.each do |queue|
97
+ next unless @queues.include? queue
70
98
 
71
- @queues.push queue
99
+ clear_limits_for_queue(queue)
100
+ @queues.delete queue
101
+ Sidekiq::Queue.delete_instance(queue)
102
+ end
72
103
  end
73
- end
74
- end
75
104
 
76
- def remove(queues)
77
- return unless queues
78
- queues.each do |queue|
79
- if @queues.include? queue
80
- clear_limits_for_queue(queue)
81
- @queues.delete queue
82
- Sidekiq::Queue.delete_instance(queue)
105
+ def handle(queues)
106
+ add(queues - @queues)
107
+ remove(@queues - queues)
83
108
  end
84
- end
85
- end
86
109
 
87
- def handle(queues)
88
- add(queues - @queues)
89
- remove(@queues - queues)
90
- end
110
+ # rubocop:disable Lint/NestedMethodDefinition
111
+ def strict_order!
112
+ @queues.uniq!
113
+ def ordered_queues
114
+ @queues
115
+ end
116
+ end
91
117
 
92
- def strict_order!
93
- @queues.uniq!
94
- def ordered_queues; @queues end
95
- end
118
+ def weighted_order!
119
+ def ordered_queues
120
+ @queues.shuffle.uniq
121
+ end
122
+ end
123
+ # rubocop:enable Lint/NestedMethodDefinition
124
+
125
+ def namespace
126
+ @namespace ||= Sidekiq.redis do |it|
127
+ if it.respond_to?(:namespace) && it.namespace
128
+ "#{it.namespace}:"
129
+ else
130
+ ''
131
+ end
132
+ end
133
+ end
96
134
 
97
- def weighted_order!
98
- def ordered_queues; @queues.shuffle.uniq end
99
- end
135
+ private
100
136
 
101
- def namespace
102
- @namespace ||= Sidekiq.redis do |it|
103
- if it.respond_to?(:namespace) and it.namespace
104
- "#{it.namespace}:"
105
- else
106
- ''
137
+ def apply_process_limit_to_queues
138
+ @queues.uniq.each do |queue_name|
139
+ apply_process_limit_to_queue(queue_name)
140
+ end
107
141
  end
108
- end
109
- end
110
-
111
- private
112
142
 
113
- def apply_process_limit_to_queues
114
- @queues.uniq.each do |queue_name|
115
- apply_process_limit_to_queue(queue_name)
116
- end
117
- end
143
+ def apply_process_limit_to_queue(queue_name)
144
+ queue = Sidekiq::Queue[queue_name]
145
+ queue.process_limit = @process_limits[queue_name.to_s] || @process_limits[queue_name.to_sym]
146
+ end
118
147
 
119
- def apply_process_limit_to_queue(queue_name)
120
- queue = Sidekiq::Queue[queue_name]
121
- queue.process_limit = @process_limits[queue_name.to_s] || @process_limits[queue_name.to_sym]
122
- end
148
+ def apply_limit_to_queues
149
+ @queues.uniq.each do |queue_name|
150
+ apply_limit_to_queue(queue_name)
151
+ end
152
+ end
123
153
 
124
- def apply_limit_to_queues
125
- @queues.uniq.each do |queue_name|
126
- apply_limit_to_queue(queue_name)
127
- end
128
- end
154
+ def apply_limit_to_queue(queue_name)
155
+ queue = Sidekiq::Queue[queue_name]
129
156
 
130
- def apply_limit_to_queue(queue_name)
131
- queue = Sidekiq::Queue[queue_name]
157
+ return if queue.limit_changed?
132
158
 
133
- unless queue.limit_changed?
134
- queue.limit = @limits[queue_name.to_s] || @limits[queue_name.to_sym]
135
- end
136
- end
159
+ queue.limit = @limits[queue_name.to_s] || @limits[queue_name.to_sym]
160
+ end
137
161
 
138
- def apply_blocks_to_queues
139
- @queues.uniq.each do |queue_name|
140
- Sidekiq::Queue[queue_name].unblock
141
- end
162
+ def apply_blocks_to_queues
163
+ @queues.uniq.each do |queue_name|
164
+ Sidekiq::Queue[queue_name].unblock
165
+ end
142
166
 
143
- @blocks.to_a.each do |it|
144
- if it.is_a? Array
145
- it.each {|name| Sidekiq::Queue[name].block_except it }
146
- else
147
- Sidekiq::Queue[it].block
167
+ @blocks.to_a.each do |it|
168
+ if it.is_a? Array
169
+ it.each { |name| Sidekiq::Queue[name].block_except it }
170
+ else
171
+ Sidekiq::Queue[it].block
172
+ end
173
+ end
148
174
  end
149
- end
150
- end
151
175
 
152
- def clear_limits_for_queue(queue_name)
153
- queue = Sidekiq::Queue[queue_name]
154
- queue.clear_limits
155
- end
176
+ def clear_limits_for_queue(queue_name)
177
+ queue = Sidekiq::Queue[queue_name]
178
+ queue.clear_limits
179
+ end
156
180
 
157
- def selector
158
- Sidekiq::LimitFetch::Global::Selector
159
- end
181
+ def selector
182
+ Sidekiq::LimitFetch::Global::Selector
183
+ end
160
184
 
161
- def saved
162
- Thread.current[THREAD_KEY]
163
- end
185
+ def saved
186
+ Thread.current[THREAD_KEY]
187
+ end
164
188
 
165
- def save(queues)
166
- Thread.current[THREAD_KEY] = queues
167
- end
189
+ def save(queues)
190
+ Thread.current[THREAD_KEY] = queues
191
+ end
168
192
 
169
- def restore
170
- saved || []
171
- ensure
172
- save nil
193
+ def restore
194
+ saved || []
195
+ ensure
196
+ save nil
197
+ end
198
+ end
173
199
  end
174
200
  end