sidekiq-limit_fetch 4.3.2 → 4.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (48) hide show
  1. checksums.yaml +4 -4
  2. data/.github/workflows/ci.yml +3 -1
  3. data/.rubocop.yml +29 -0
  4. data/Appraisals +6 -0
  5. data/Gemfile +2 -0
  6. data/README.md +11 -1
  7. data/Rakefile +2 -0
  8. data/bench/compare.rb +17 -13
  9. data/demo/Gemfile +3 -2
  10. data/demo/Rakefile +6 -5
  11. data/demo/app/workers/a_worker.rb +2 -0
  12. data/demo/app/workers/b_worker.rb +2 -0
  13. data/demo/app/workers/c_worker.rb +2 -1
  14. data/demo/app/workers/fast_worker.rb +2 -0
  15. data/demo/app/workers/slow_worker.rb +2 -0
  16. data/demo/config/application.rb +3 -1
  17. data/demo/config/boot.rb +4 -2
  18. data/demo/config/environment.rb +3 -1
  19. data/demo/config/environments/development.rb +2 -0
  20. data/docker-compose.dev.yml +13 -0
  21. data/gemfiles/sidekiq_6.0.gemfile.lock +3 -4
  22. data/gemfiles/sidekiq_6.1.gemfile.lock +3 -4
  23. data/gemfiles/sidekiq_6.2.gemfile.lock +3 -4
  24. data/gemfiles/sidekiq_6.3.gemfile.lock +3 -4
  25. data/gemfiles/sidekiq_6.4.gemfile.lock +3 -4
  26. data/gemfiles/sidekiq_6.5.gemfile.lock +3 -4
  27. data/gemfiles/sidekiq_7.0.gemfile +7 -0
  28. data/gemfiles/sidekiq_7.0.gemfile.lock +58 -0
  29. data/gemfiles/sidekiq_master.gemfile.lock +3 -4
  30. data/lib/sidekiq/extensions/manager.rb +21 -13
  31. data/lib/sidekiq/extensions/queue.rb +16 -13
  32. data/lib/sidekiq/limit_fetch/global/monitor.rb +64 -53
  33. data/lib/sidekiq/limit_fetch/global/selector.rb +49 -40
  34. data/lib/sidekiq/limit_fetch/global/semaphore.rb +130 -123
  35. data/lib/sidekiq/limit_fetch/instances.rb +22 -16
  36. data/lib/sidekiq/limit_fetch/queues.rb +165 -124
  37. data/lib/sidekiq/limit_fetch/unit_of_work.rb +26 -22
  38. data/lib/sidekiq/limit_fetch.rb +73 -54
  39. data/lib/sidekiq-limit_fetch.rb +2 -0
  40. data/sidekiq-limit_fetch.gemspec +20 -13
  41. data/spec/sidekiq/extensions/manager_spec.rb +19 -0
  42. data/spec/sidekiq/extensions/queue_spec.rb +2 -0
  43. data/spec/sidekiq/limit_fetch/global/monitor_spec.rb +86 -5
  44. data/spec/sidekiq/limit_fetch/queues_spec.rb +34 -18
  45. data/spec/sidekiq/limit_fetch/semaphore_spec.rb +2 -0
  46. data/spec/sidekiq/limit_fetch_spec.rb +14 -4
  47. data/spec/spec_helper.rb +15 -4
  48. metadata +34 -21
@@ -1,72 +1,83 @@
1
- module Sidekiq::LimitFetch::Global
2
- module Monitor
3
- extend self
1
+ # frozen_string_literal: true
4
2
 
5
- HEARTBEAT_PREFIX = 'limit:heartbeat:'
6
- PROCESS_SET = 'limit:processes'
7
- HEARTBEAT_TTL = 20
8
- REFRESH_TIMEOUT = 5
3
+ module Sidekiq
4
+ module LimitFetch
5
+ module Global
6
+ module Monitor
7
+ extend self
9
8
 
10
- def start!(ttl=HEARTBEAT_TTL, timeout=REFRESH_TIMEOUT)
11
- Thread.new do
12
- loop do
13
- Sidekiq::LimitFetch.redis_retryable do
14
- handle_dynamic_queues
15
- update_heartbeat ttl
16
- invalidate_old_processes
9
+ HEARTBEAT_PREFIX = 'limit:heartbeat:'
10
+ PROCESS_SET = 'limit:processes'
11
+ HEARTBEAT_TTL = 20
12
+ REFRESH_TIMEOUT = 5
13
+
14
+ def start!(ttl = HEARTBEAT_TTL, timeout = REFRESH_TIMEOUT)
15
+ Thread.new do
16
+ loop do
17
+ Sidekiq::LimitFetch.redis_retryable do
18
+ handle_dynamic_queues
19
+ update_heartbeat ttl
20
+ invalidate_old_processes
21
+ end
22
+
23
+ sleep timeout
24
+ end
17
25
  end
26
+ end
18
27
 
19
- sleep timeout
28
+ def all_processes
29
+ Sidekiq.redis { |it| it.smembers PROCESS_SET }
20
30
  end
21
- end
22
- end
23
31
 
24
- def all_processes
25
- Sidekiq.redis {|it| it.smembers PROCESS_SET }
26
- end
32
+ def old_processes
33
+ all_processes.reject do |process|
34
+ Sidekiq.redis { |it| it.get heartbeat_key process } == '1'
35
+ end
36
+ end
27
37
 
28
- def old_processes
29
- all_processes.reject do |process|
30
- Sidekiq.redis {|it| it.get heartbeat_key process }
31
- end
32
- end
38
+ def remove_old_processes!
39
+ Sidekiq.redis do |it|
40
+ old_processes.each { |process| it.srem PROCESS_SET, [process] }
41
+ end
42
+ end
33
43
 
34
- def remove_old_processes!
35
- Sidekiq.redis do |it|
36
- old_processes.each {|process| it.srem PROCESS_SET, [process] }
37
- end
38
- end
44
+ def handle_dynamic_queues
45
+ queues = Sidekiq::LimitFetch::Queues
46
+ return unless queues.dynamic?
39
47
 
40
- def handle_dynamic_queues
41
- queues = Sidekiq::LimitFetch::Queues
42
- queues.handle Sidekiq::Queue.all.map(&:name) if queues.dynamic?
43
- end
48
+ available_queues = Sidekiq::Queue.all.map(&:name).reject do |it|
49
+ queues.dynamic_exclude.include? it
50
+ end
51
+ queues.handle available_queues
52
+ end
44
53
 
45
- private
54
+ private
46
55
 
47
- def update_heartbeat(ttl)
48
- Sidekiq.redis do |it|
49
- it.multi do |pipeline|
50
- pipeline.set heartbeat_key, true
51
- pipeline.sadd PROCESS_SET, [Selector.uuid]
52
- pipeline.expire heartbeat_key, ttl
56
+ def update_heartbeat(ttl)
57
+ Sidekiq.redis do |it|
58
+ it.multi do |pipeline|
59
+ pipeline.set heartbeat_key, '1'
60
+ pipeline.sadd PROCESS_SET, [Selector.uuid]
61
+ pipeline.expire heartbeat_key, ttl
62
+ end
63
+ end
53
64
  end
54
- end
55
- end
56
65
 
57
- def invalidate_old_processes
58
- Sidekiq.redis do |it|
59
- remove_old_processes!
60
- processes = all_processes
66
+ def invalidate_old_processes
67
+ Sidekiq.redis do |_it|
68
+ remove_old_processes!
69
+ processes = all_processes
61
70
 
62
- Sidekiq::Queue.instances.each do |queue|
63
- queue.remove_locks_except! processes
71
+ Sidekiq::Queue.instances.each do |queue|
72
+ queue.remove_locks_except! processes
73
+ end
74
+ end
64
75
  end
65
- end
66
- end
67
76
 
68
- def heartbeat_key(process=Selector.uuid)
69
- HEARTBEAT_PREFIX + process
77
+ def heartbeat_key(process = Selector.uuid)
78
+ HEARTBEAT_PREFIX + process
79
+ end
80
+ end
70
81
  end
71
82
  end
72
83
  end
@@ -1,48 +1,55 @@
1
- module Sidekiq::LimitFetch::Global
2
- module Selector
3
- extend self
1
+ # frozen_string_literal: true
4
2
 
5
- MUTEX_FOR_UUID = Mutex.new
3
+ module Sidekiq
4
+ module LimitFetch
5
+ module Global
6
+ module Selector
7
+ extend self
6
8
 
7
- def acquire(queues, namespace)
8
- redis_eval :acquire, [namespace, uuid, queues]
9
- end
9
+ MUTEX_FOR_UUID = Mutex.new
10
10
 
11
- def release(queues, namespace)
12
- redis_eval :release, [namespace, uuid, queues]
13
- end
11
+ def acquire(queues, namespace)
12
+ redis_eval :acquire, [namespace, uuid, queues]
13
+ end
14
14
 
15
- def uuid
16
- # - if we'll remove "@uuid ||=" from inside of mutex
17
- # then @uuid can be overwritten
18
- # - if we'll remove "@uuid ||=" from outside of mutex
19
- # then each read will lead to mutex
20
- @uuid ||= MUTEX_FOR_UUID.synchronize { @uuid || SecureRandom.uuid }
21
- end
15
+ def release(queues, namespace)
16
+ redis_eval :release, [namespace, uuid, queues]
17
+ end
22
18
 
23
- private
19
+ def uuid
20
+ # - if we'll remove "@uuid ||=" from inside of mutex
21
+ # then @uuid can be overwritten
22
+ # - if we'll remove "@uuid ||=" from outside of mutex
23
+ # then each read will lead to mutex
24
+ @uuid ||= MUTEX_FOR_UUID.synchronize { @uuid || SecureRandom.uuid }
25
+ end
26
+
27
+ private
28
+
29
+ def redis_eval(script_name, args)
30
+ Sidekiq.redis do |it|
31
+ it.evalsha send("redis_#{script_name}_sha"), [], args
32
+ rescue Sidekiq::LimitFetch::RedisCommandError => e
33
+ raise unless e.message.include? 'NOSCRIPT'
24
34
 
25
- def redis_eval(script_name, args)
26
- Sidekiq.redis do |it|
27
- begin
28
- it.evalsha send("redis_#{script_name}_sha"), argv: args
29
- rescue Redis::CommandError => error
30
- raise unless error.message.include? 'NOSCRIPT'
31
- it.eval send("redis_#{script_name}_script"), argv: args
35
+ if Sidekiq::LimitFetch.post_7?
36
+ it.eval send("redis_#{script_name}_script"), 0, *args
37
+ else
38
+ it.eval send("redis_#{script_name}_script"), argv: args
39
+ end
40
+ end
32
41
  end
33
- end
34
- end
35
42
 
36
- def redis_acquire_sha
37
- @acquire_sha ||= Digest::SHA1.hexdigest redis_acquire_script
38
- end
43
+ def redis_acquire_sha
44
+ @redis_acquire_sha ||= OpenSSL::Digest::SHA1.hexdigest redis_acquire_script
45
+ end
39
46
 
40
- def redis_release_sha
41
- @release_sha ||= Digest::SHA1.hexdigest redis_release_script
42
- end
47
+ def redis_release_sha
48
+ @redis_release_sha ||= OpenSSL::Digest::SHA1.hexdigest redis_release_script
49
+ end
43
50
 
44
- def redis_acquire_script
45
- <<-LUA
51
+ def redis_acquire_script
52
+ <<-LUA
46
53
  local namespace = table.remove(ARGV, 1)..'limit_fetch:'
47
54
  local worker_name = table.remove(ARGV, 1)
48
55
  local queues = ARGV
@@ -106,11 +113,11 @@ module Sidekiq::LimitFetch::Global
106
113
  end
107
114
 
108
115
  return available
109
- LUA
110
- end
116
+ LUA
117
+ end
111
118
 
112
- def redis_release_script
113
- <<-LUA
119
+ def redis_release_script
120
+ <<-LUA
114
121
  local namespace = table.remove(ARGV, 1)..'limit_fetch:'
115
122
  local worker_name = table.remove(ARGV, 1)
116
123
  local queues = ARGV
@@ -119,7 +126,9 @@ module Sidekiq::LimitFetch::Global
119
126
  local probed_key = namespace..'probed:'..queue
120
127
  redis.call('lrem', probed_key, 1, worker_name)
121
128
  end
122
- LUA
129
+ LUA
130
+ end
131
+ end
123
132
  end
124
133
  end
125
134
  end
@@ -1,136 +1,141 @@
1
- module Sidekiq::LimitFetch::Global
2
- class Semaphore
3
- PREFIX = 'limit_fetch'
1
+ # frozen_string_literal: true
4
2
 
5
- attr_reader :local_busy
3
+ module Sidekiq
4
+ module LimitFetch
5
+ module Global
6
+ class Semaphore
7
+ PREFIX = 'limit_fetch'
6
8
 
7
- def initialize(name)
8
- @name = name
9
- @lock = Mutex.new
10
- @local_busy = 0
11
- end
9
+ attr_reader :local_busy
12
10
 
13
- def limit
14
- value = redis {|it| it.get "#{PREFIX}:limit:#@name" }
15
- value.to_i if value
16
- end
11
+ def initialize(name)
12
+ @name = name
13
+ @lock = Mutex.new
14
+ @local_busy = 0
15
+ end
17
16
 
18
- def limit=(value)
19
- @limit_changed = true
17
+ def limit
18
+ value = redis { |it| it.get "#{PREFIX}:limit:#{@name}" }
19
+ value&.to_i
20
+ end
20
21
 
21
- if value
22
- redis {|it| it.set "#{PREFIX}:limit:#@name", value }
23
- else
24
- redis {|it| it.del "#{PREFIX}:limit:#@name" }
25
- end
26
- end
22
+ def limit=(value)
23
+ @limit_changed = true
27
24
 
28
- def limit_changed?
29
- @limit_changed
30
- end
25
+ if value
26
+ redis { |it| it.set "#{PREFIX}:limit:#{@name}", value }
27
+ else
28
+ redis { |it| it.del "#{PREFIX}:limit:#{@name}" }
29
+ end
30
+ end
31
31
 
32
- def process_limit
33
- value = redis {|it| it.get "#{PREFIX}:process_limit:#@name" }
34
- value.to_i if value
35
- end
32
+ def limit_changed?
33
+ @limit_changed
34
+ end
36
35
 
37
- def process_limit=(value)
38
- if value
39
- redis {|it| it.set "#{PREFIX}:process_limit:#@name", value }
40
- else
41
- redis {|it| it.del "#{PREFIX}:process_limit:#@name" }
42
- end
43
- end
36
+ def process_limit
37
+ value = redis { |it| it.get "#{PREFIX}:process_limit:#{@name}" }
38
+ value&.to_i
39
+ end
44
40
 
45
- def acquire
46
- Selector.acquire([@name], namespace).size > 0
47
- end
41
+ def process_limit=(value)
42
+ if value
43
+ redis { |it| it.set "#{PREFIX}:process_limit:#{@name}", value }
44
+ else
45
+ redis { |it| it.del "#{PREFIX}:process_limit:#{@name}" }
46
+ end
47
+ end
48
48
 
49
- def release
50
- redis {|it| it.lrem "#{PREFIX}:probed:#@name", 1, Selector.uuid }
51
- end
49
+ def acquire
50
+ Selector.acquire([@name], namespace).size.positive?
51
+ end
52
52
 
53
- def busy
54
- redis {|it| it.llen "#{PREFIX}:busy:#@name" }
55
- end
53
+ def release
54
+ redis { |it| it.lrem "#{PREFIX}:probed:#{@name}", 1, Selector.uuid }
55
+ end
56
56
 
57
- def busy_processes
58
- redis {|it| it.lrange "#{PREFIX}:busy:#@name", 0, -1 }
59
- end
57
+ def busy
58
+ redis { |it| it.llen "#{PREFIX}:busy:#{@name}" }
59
+ end
60
60
 
61
- def increase_busy
62
- increase_local_busy
63
- redis {|it| it.rpush "#{PREFIX}:busy:#@name", Selector.uuid }
64
- end
61
+ def busy_processes
62
+ redis { |it| it.lrange "#{PREFIX}:busy:#{@name}", 0, -1 }
63
+ end
65
64
 
66
- def decrease_busy
67
- decrease_local_busy
68
- redis {|it| it.lrem "#{PREFIX}:busy:#@name", 1, Selector.uuid }
69
- end
65
+ def increase_busy
66
+ increase_local_busy
67
+ redis { |it| it.rpush "#{PREFIX}:busy:#{@name}", Selector.uuid }
68
+ end
70
69
 
71
- def probed
72
- redis {|it| it.llen "#{PREFIX}:probed:#@name" }
73
- end
70
+ def decrease_busy
71
+ decrease_local_busy
72
+ redis { |it| it.lrem "#{PREFIX}:busy:#{@name}", 1, Selector.uuid }
73
+ end
74
74
 
75
- def probed_processes
76
- redis {|it| it.lrange "#{PREFIX}:probed:#@name", 0, -1 }
77
- end
75
+ def probed
76
+ redis { |it| it.llen "#{PREFIX}:probed:#{@name}" }
77
+ end
78
78
 
79
- def pause
80
- redis {|it| it.set "#{PREFIX}:pause:#@name", true }
81
- end
79
+ def probed_processes
80
+ redis { |it| it.lrange "#{PREFIX}:probed:#{@name}", 0, -1 }
81
+ end
82
82
 
83
- def pause_for_ms ms
84
- redis {|it| it.psetex "#{PREFIX}:pause:#@name", ms, true }
85
- end
83
+ def pause
84
+ redis { |it| it.set "#{PREFIX}:pause:#{@name}", '1' }
85
+ end
86
86
 
87
- def unpause
88
- redis {|it| it.del "#{PREFIX}:pause:#@name" }
89
- end
87
+ def pause_for_ms(milliseconds)
88
+ redis { |it| it.psetex "#{PREFIX}:pause:#{@name}", milliseconds, 1 }
89
+ end
90
90
 
91
- def paused?
92
- redis {|it| it.get "#{PREFIX}:pause:#@name" }
93
- end
91
+ def unpause
92
+ redis { |it| it.del "#{PREFIX}:pause:#{@name}" }
93
+ end
94
94
 
95
- def block
96
- redis {|it| it.set "#{PREFIX}:block:#@name", true }
97
- end
95
+ def paused?
96
+ redis { |it| it.get "#{PREFIX}:pause:#{@name}" } == '1'
97
+ end
98
98
 
99
- def block_except(*queues)
100
- raise ArgumentError if queues.empty?
101
- redis {|it| it.set "#{PREFIX}:block:#@name", queues.join(',') }
102
- end
99
+ def block
100
+ redis { |it| it.set "#{PREFIX}:block:#{@name}", '1' }
101
+ end
103
102
 
104
- def unblock
105
- redis {|it| it.del "#{PREFIX}:block:#@name" }
106
- end
103
+ def block_except(*queues)
104
+ raise ArgumentError if queues.empty?
107
105
 
108
- def blocking?
109
- redis {|it| it.get "#{PREFIX}:block:#@name" }
110
- end
106
+ redis { |it| it.set "#{PREFIX}:block:#{@name}", queues.join(',') }
107
+ end
111
108
 
112
- def clear_limits
113
- redis do |it|
114
- %w(block busy limit pause probed process_limit).each do |key|
115
- it.del "#{PREFIX}:#{key}:#@name"
109
+ def unblock
110
+ redis { |it| it.del "#{PREFIX}:block:#{@name}" }
116
111
  end
117
- end
118
- end
119
112
 
120
- def increase_local_busy
121
- @lock.synchronize { @local_busy += 1 }
122
- end
113
+ def blocking?
114
+ redis { |it| it.get "#{PREFIX}:block:#{@name}" } == '1'
115
+ end
123
116
 
124
- def decrease_local_busy
125
- @lock.synchronize { @local_busy -= 1 }
126
- end
117
+ def clear_limits
118
+ redis do |it|
119
+ %w[block busy limit pause probed process_limit].each do |key|
120
+ it.del "#{PREFIX}:#{key}:#{@name}"
121
+ end
122
+ end
123
+ end
127
124
 
128
- def local_busy?
129
- @local_busy > 0
130
- end
125
+ def increase_local_busy
126
+ @lock.synchronize { @local_busy += 1 }
127
+ end
128
+
129
+ def decrease_local_busy
130
+ @lock.synchronize { @local_busy -= 1 }
131
+ end
132
+
133
+ def local_busy?
134
+ @local_busy.positive?
135
+ end
131
136
 
132
- def explain
133
- <<-END.gsub(/^ {8}/, '')
137
+ def explain
138
+ <<-INFO.gsub(/^ {8}/, '')
134
139
  Current sidekiq process: #{Selector.uuid}
135
140
 
136
141
  All processes:
@@ -153,31 +158,33 @@ module Sidekiq::LimitFetch::Global
153
158
 
154
159
  Blocking:
155
160
  #{blocking?}
156
- END
157
- end
161
+ INFO
162
+ end
158
163
 
159
- def remove_locks_except!(processes)
160
- locked_processes = probed_processes.uniq
161
- (locked_processes - processes).each do |dead_process|
162
- remove_lock! dead_process
163
- end
164
- end
164
+ def remove_locks_except!(processes)
165
+ locked_processes = probed_processes.uniq
166
+ (locked_processes - processes).each do |dead_process|
167
+ remove_lock! dead_process
168
+ end
169
+ end
165
170
 
166
- def remove_lock!(process)
167
- redis do |it|
168
- it.lrem "#{PREFIX}:probed:#@name", 0, process
169
- it.lrem "#{PREFIX}:busy:#@name", 0, process
170
- end
171
- end
171
+ def remove_lock!(process)
172
+ redis do |it|
173
+ it.lrem "#{PREFIX}:probed:#{@name}", 0, process
174
+ it.lrem "#{PREFIX}:busy:#{@name}", 0, process
175
+ end
176
+ end
172
177
 
173
- private
178
+ private
174
179
 
175
- def redis(&block)
176
- Sidekiq.redis(&block)
177
- end
180
+ def redis(&block)
181
+ Sidekiq.redis(&block)
182
+ end
178
183
 
179
- def namespace
180
- Sidekiq::LimitFetch::Queues.namespace
184
+ def namespace
185
+ Sidekiq::LimitFetch::Queues.namespace
186
+ end
187
+ end
181
188
  end
182
189
  end
183
190
  end
@@ -1,23 +1,29 @@
1
- module Sidekiq::LimitFetch::Instances
2
- def self.extended(klass)
3
- klass.instance_variable_set :@instances, {}
4
- end
1
+ # frozen_string_literal: true
5
2
 
6
- def new(*args)
7
- @instances[args] ||= super
8
- end
3
+ module Sidekiq
4
+ module LimitFetch
5
+ module Instances
6
+ def self.extended(klass)
7
+ klass.instance_variable_set :@instances, {}
8
+ end
9
9
 
10
- alias [] new
10
+ def new(*args)
11
+ @instances[args] ||= super
12
+ end
11
13
 
12
- def instances
13
- @instances.values
14
- end
14
+ alias [] new
15
15
 
16
- def reset_instances!
17
- @instances = {}
18
- end
16
+ def instances
17
+ @instances.values
18
+ end
19
+
20
+ def reset_instances!
21
+ @instances = {}
22
+ end
19
23
 
20
- def delete_instance(name)
21
- @instances.delete [name]
24
+ def delete_instance(name)
25
+ @instances.delete [name]
26
+ end
27
+ end
22
28
  end
23
29
  end