sidekiq-limit_fetch 4.3.2 → 4.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.github/workflows/ci.yml +3 -1
- data/.rubocop.yml +29 -0
- data/Appraisals +6 -0
- data/Gemfile +2 -0
- data/README.md +11 -1
- data/Rakefile +2 -0
- data/bench/compare.rb +17 -13
- data/demo/Gemfile +3 -2
- data/demo/Rakefile +6 -5
- data/demo/app/workers/a_worker.rb +2 -0
- data/demo/app/workers/b_worker.rb +2 -0
- data/demo/app/workers/c_worker.rb +2 -1
- data/demo/app/workers/fast_worker.rb +2 -0
- data/demo/app/workers/slow_worker.rb +2 -0
- data/demo/config/application.rb +3 -1
- data/demo/config/boot.rb +4 -2
- data/demo/config/environment.rb +3 -1
- data/demo/config/environments/development.rb +2 -0
- data/docker-compose.dev.yml +13 -0
- data/gemfiles/sidekiq_6.0.gemfile.lock +3 -4
- data/gemfiles/sidekiq_6.1.gemfile.lock +3 -4
- data/gemfiles/sidekiq_6.2.gemfile.lock +3 -4
- data/gemfiles/sidekiq_6.3.gemfile.lock +3 -4
- data/gemfiles/sidekiq_6.4.gemfile.lock +3 -4
- data/gemfiles/sidekiq_6.5.gemfile.lock +3 -4
- data/gemfiles/sidekiq_7.0.gemfile +7 -0
- data/gemfiles/sidekiq_7.0.gemfile.lock +58 -0
- data/gemfiles/sidekiq_master.gemfile.lock +3 -4
- data/lib/sidekiq/extensions/manager.rb +21 -13
- data/lib/sidekiq/extensions/queue.rb +16 -13
- data/lib/sidekiq/limit_fetch/global/monitor.rb +64 -53
- data/lib/sidekiq/limit_fetch/global/selector.rb +49 -40
- data/lib/sidekiq/limit_fetch/global/semaphore.rb +130 -123
- data/lib/sidekiq/limit_fetch/instances.rb +22 -16
- data/lib/sidekiq/limit_fetch/queues.rb +165 -124
- data/lib/sidekiq/limit_fetch/unit_of_work.rb +26 -22
- data/lib/sidekiq/limit_fetch.rb +73 -54
- data/lib/sidekiq-limit_fetch.rb +2 -0
- data/sidekiq-limit_fetch.gemspec +20 -13
- data/spec/sidekiq/extensions/manager_spec.rb +19 -0
- data/spec/sidekiq/extensions/queue_spec.rb +2 -0
- data/spec/sidekiq/limit_fetch/global/monitor_spec.rb +86 -5
- data/spec/sidekiq/limit_fetch/queues_spec.rb +34 -18
- data/spec/sidekiq/limit_fetch/semaphore_spec.rb +2 -0
- data/spec/sidekiq/limit_fetch_spec.rb +14 -4
- data/spec/spec_helper.rb +15 -4
- metadata +34 -21
@@ -1,72 +1,83 @@
|
|
1
|
-
|
2
|
-
module Monitor
|
3
|
-
extend self
|
1
|
+
# frozen_string_literal: true
|
4
2
|
|
5
|
-
|
6
|
-
|
7
|
-
|
8
|
-
|
3
|
+
module Sidekiq
|
4
|
+
module LimitFetch
|
5
|
+
module Global
|
6
|
+
module Monitor
|
7
|
+
extend self
|
9
8
|
|
10
|
-
|
11
|
-
|
12
|
-
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
9
|
+
HEARTBEAT_PREFIX = 'limit:heartbeat:'
|
10
|
+
PROCESS_SET = 'limit:processes'
|
11
|
+
HEARTBEAT_TTL = 20
|
12
|
+
REFRESH_TIMEOUT = 5
|
13
|
+
|
14
|
+
def start!(ttl = HEARTBEAT_TTL, timeout = REFRESH_TIMEOUT)
|
15
|
+
Thread.new do
|
16
|
+
loop do
|
17
|
+
Sidekiq::LimitFetch.redis_retryable do
|
18
|
+
handle_dynamic_queues
|
19
|
+
update_heartbeat ttl
|
20
|
+
invalidate_old_processes
|
21
|
+
end
|
22
|
+
|
23
|
+
sleep timeout
|
24
|
+
end
|
17
25
|
end
|
26
|
+
end
|
18
27
|
|
19
|
-
|
28
|
+
def all_processes
|
29
|
+
Sidekiq.redis { |it| it.smembers PROCESS_SET }
|
20
30
|
end
|
21
|
-
end
|
22
|
-
end
|
23
31
|
|
24
|
-
|
25
|
-
|
26
|
-
|
32
|
+
def old_processes
|
33
|
+
all_processes.reject do |process|
|
34
|
+
Sidekiq.redis { |it| it.get heartbeat_key process } == '1'
|
35
|
+
end
|
36
|
+
end
|
27
37
|
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
|
32
|
-
|
38
|
+
def remove_old_processes!
|
39
|
+
Sidekiq.redis do |it|
|
40
|
+
old_processes.each { |process| it.srem PROCESS_SET, [process] }
|
41
|
+
end
|
42
|
+
end
|
33
43
|
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
end
|
38
|
-
end
|
44
|
+
def handle_dynamic_queues
|
45
|
+
queues = Sidekiq::LimitFetch::Queues
|
46
|
+
return unless queues.dynamic?
|
39
47
|
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
48
|
+
available_queues = Sidekiq::Queue.all.map(&:name).reject do |it|
|
49
|
+
queues.dynamic_exclude.include? it
|
50
|
+
end
|
51
|
+
queues.handle available_queues
|
52
|
+
end
|
44
53
|
|
45
|
-
|
54
|
+
private
|
46
55
|
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
|
51
|
-
|
52
|
-
|
56
|
+
def update_heartbeat(ttl)
|
57
|
+
Sidekiq.redis do |it|
|
58
|
+
it.multi do |pipeline|
|
59
|
+
pipeline.set heartbeat_key, '1'
|
60
|
+
pipeline.sadd PROCESS_SET, [Selector.uuid]
|
61
|
+
pipeline.expire heartbeat_key, ttl
|
62
|
+
end
|
63
|
+
end
|
53
64
|
end
|
54
|
-
end
|
55
|
-
end
|
56
65
|
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
66
|
+
def invalidate_old_processes
|
67
|
+
Sidekiq.redis do |_it|
|
68
|
+
remove_old_processes!
|
69
|
+
processes = all_processes
|
61
70
|
|
62
|
-
|
63
|
-
|
71
|
+
Sidekiq::Queue.instances.each do |queue|
|
72
|
+
queue.remove_locks_except! processes
|
73
|
+
end
|
74
|
+
end
|
64
75
|
end
|
65
|
-
end
|
66
|
-
end
|
67
76
|
|
68
|
-
|
69
|
-
|
77
|
+
def heartbeat_key(process = Selector.uuid)
|
78
|
+
HEARTBEAT_PREFIX + process
|
79
|
+
end
|
80
|
+
end
|
70
81
|
end
|
71
82
|
end
|
72
83
|
end
|
@@ -1,48 +1,55 @@
|
|
1
|
-
|
2
|
-
module Selector
|
3
|
-
extend self
|
1
|
+
# frozen_string_literal: true
|
4
2
|
|
5
|
-
|
3
|
+
module Sidekiq
|
4
|
+
module LimitFetch
|
5
|
+
module Global
|
6
|
+
module Selector
|
7
|
+
extend self
|
6
8
|
|
7
|
-
|
8
|
-
redis_eval :acquire, [namespace, uuid, queues]
|
9
|
-
end
|
9
|
+
MUTEX_FOR_UUID = Mutex.new
|
10
10
|
|
11
|
-
|
12
|
-
|
13
|
-
|
11
|
+
def acquire(queues, namespace)
|
12
|
+
redis_eval :acquire, [namespace, uuid, queues]
|
13
|
+
end
|
14
14
|
|
15
|
-
|
16
|
-
|
17
|
-
|
18
|
-
# - if we'll remove "@uuid ||=" from outside of mutex
|
19
|
-
# then each read will lead to mutex
|
20
|
-
@uuid ||= MUTEX_FOR_UUID.synchronize { @uuid || SecureRandom.uuid }
|
21
|
-
end
|
15
|
+
def release(queues, namespace)
|
16
|
+
redis_eval :release, [namespace, uuid, queues]
|
17
|
+
end
|
22
18
|
|
23
|
-
|
19
|
+
def uuid
|
20
|
+
# - if we'll remove "@uuid ||=" from inside of mutex
|
21
|
+
# then @uuid can be overwritten
|
22
|
+
# - if we'll remove "@uuid ||=" from outside of mutex
|
23
|
+
# then each read will lead to mutex
|
24
|
+
@uuid ||= MUTEX_FOR_UUID.synchronize { @uuid || SecureRandom.uuid }
|
25
|
+
end
|
26
|
+
|
27
|
+
private
|
28
|
+
|
29
|
+
def redis_eval(script_name, args)
|
30
|
+
Sidekiq.redis do |it|
|
31
|
+
it.evalsha send("redis_#{script_name}_sha"), [], args
|
32
|
+
rescue Sidekiq::LimitFetch::RedisCommandError => e
|
33
|
+
raise unless e.message.include? 'NOSCRIPT'
|
24
34
|
|
25
|
-
|
26
|
-
|
27
|
-
|
28
|
-
|
29
|
-
|
30
|
-
|
31
|
-
it.eval send("redis_#{script_name}_script"), argv: args
|
35
|
+
if Sidekiq::LimitFetch.post_7?
|
36
|
+
it.eval send("redis_#{script_name}_script"), 0, *args
|
37
|
+
else
|
38
|
+
it.eval send("redis_#{script_name}_script"), argv: args
|
39
|
+
end
|
40
|
+
end
|
32
41
|
end
|
33
|
-
end
|
34
|
-
end
|
35
42
|
|
36
|
-
|
37
|
-
|
38
|
-
|
43
|
+
def redis_acquire_sha
|
44
|
+
@redis_acquire_sha ||= OpenSSL::Digest::SHA1.hexdigest redis_acquire_script
|
45
|
+
end
|
39
46
|
|
40
|
-
|
41
|
-
|
42
|
-
|
47
|
+
def redis_release_sha
|
48
|
+
@redis_release_sha ||= OpenSSL::Digest::SHA1.hexdigest redis_release_script
|
49
|
+
end
|
43
50
|
|
44
|
-
|
45
|
-
|
51
|
+
def redis_acquire_script
|
52
|
+
<<-LUA
|
46
53
|
local namespace = table.remove(ARGV, 1)..'limit_fetch:'
|
47
54
|
local worker_name = table.remove(ARGV, 1)
|
48
55
|
local queues = ARGV
|
@@ -106,11 +113,11 @@ module Sidekiq::LimitFetch::Global
|
|
106
113
|
end
|
107
114
|
|
108
115
|
return available
|
109
|
-
|
110
|
-
|
116
|
+
LUA
|
117
|
+
end
|
111
118
|
|
112
|
-
|
113
|
-
|
119
|
+
def redis_release_script
|
120
|
+
<<-LUA
|
114
121
|
local namespace = table.remove(ARGV, 1)..'limit_fetch:'
|
115
122
|
local worker_name = table.remove(ARGV, 1)
|
116
123
|
local queues = ARGV
|
@@ -119,7 +126,9 @@ module Sidekiq::LimitFetch::Global
|
|
119
126
|
local probed_key = namespace..'probed:'..queue
|
120
127
|
redis.call('lrem', probed_key, 1, worker_name)
|
121
128
|
end
|
122
|
-
|
129
|
+
LUA
|
130
|
+
end
|
131
|
+
end
|
123
132
|
end
|
124
133
|
end
|
125
134
|
end
|
@@ -1,136 +1,141 @@
|
|
1
|
-
|
2
|
-
class Semaphore
|
3
|
-
PREFIX = 'limit_fetch'
|
1
|
+
# frozen_string_literal: true
|
4
2
|
|
5
|
-
|
3
|
+
module Sidekiq
|
4
|
+
module LimitFetch
|
5
|
+
module Global
|
6
|
+
class Semaphore
|
7
|
+
PREFIX = 'limit_fetch'
|
6
8
|
|
7
|
-
|
8
|
-
@name = name
|
9
|
-
@lock = Mutex.new
|
10
|
-
@local_busy = 0
|
11
|
-
end
|
9
|
+
attr_reader :local_busy
|
12
10
|
|
13
|
-
|
14
|
-
|
15
|
-
|
16
|
-
|
11
|
+
def initialize(name)
|
12
|
+
@name = name
|
13
|
+
@lock = Mutex.new
|
14
|
+
@local_busy = 0
|
15
|
+
end
|
17
16
|
|
18
|
-
|
19
|
-
|
17
|
+
def limit
|
18
|
+
value = redis { |it| it.get "#{PREFIX}:limit:#{@name}" }
|
19
|
+
value&.to_i
|
20
|
+
end
|
20
21
|
|
21
|
-
|
22
|
-
|
23
|
-
else
|
24
|
-
redis {|it| it.del "#{PREFIX}:limit:#@name" }
|
25
|
-
end
|
26
|
-
end
|
22
|
+
def limit=(value)
|
23
|
+
@limit_changed = true
|
27
24
|
|
28
|
-
|
29
|
-
|
30
|
-
|
25
|
+
if value
|
26
|
+
redis { |it| it.set "#{PREFIX}:limit:#{@name}", value }
|
27
|
+
else
|
28
|
+
redis { |it| it.del "#{PREFIX}:limit:#{@name}" }
|
29
|
+
end
|
30
|
+
end
|
31
31
|
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
end
|
32
|
+
def limit_changed?
|
33
|
+
@limit_changed
|
34
|
+
end
|
36
35
|
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
redis {|it| it.del "#{PREFIX}:process_limit:#@name" }
|
42
|
-
end
|
43
|
-
end
|
36
|
+
def process_limit
|
37
|
+
value = redis { |it| it.get "#{PREFIX}:process_limit:#{@name}" }
|
38
|
+
value&.to_i
|
39
|
+
end
|
44
40
|
|
45
|
-
|
46
|
-
|
47
|
-
|
41
|
+
def process_limit=(value)
|
42
|
+
if value
|
43
|
+
redis { |it| it.set "#{PREFIX}:process_limit:#{@name}", value }
|
44
|
+
else
|
45
|
+
redis { |it| it.del "#{PREFIX}:process_limit:#{@name}" }
|
46
|
+
end
|
47
|
+
end
|
48
48
|
|
49
|
-
|
50
|
-
|
51
|
-
|
49
|
+
def acquire
|
50
|
+
Selector.acquire([@name], namespace).size.positive?
|
51
|
+
end
|
52
52
|
|
53
|
-
|
54
|
-
|
55
|
-
|
53
|
+
def release
|
54
|
+
redis { |it| it.lrem "#{PREFIX}:probed:#{@name}", 1, Selector.uuid }
|
55
|
+
end
|
56
56
|
|
57
|
-
|
58
|
-
|
59
|
-
|
57
|
+
def busy
|
58
|
+
redis { |it| it.llen "#{PREFIX}:busy:#{@name}" }
|
59
|
+
end
|
60
60
|
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
end
|
61
|
+
def busy_processes
|
62
|
+
redis { |it| it.lrange "#{PREFIX}:busy:#{@name}", 0, -1 }
|
63
|
+
end
|
65
64
|
|
66
|
-
|
67
|
-
|
68
|
-
|
69
|
-
|
65
|
+
def increase_busy
|
66
|
+
increase_local_busy
|
67
|
+
redis { |it| it.rpush "#{PREFIX}:busy:#{@name}", Selector.uuid }
|
68
|
+
end
|
70
69
|
|
71
|
-
|
72
|
-
|
73
|
-
|
70
|
+
def decrease_busy
|
71
|
+
decrease_local_busy
|
72
|
+
redis { |it| it.lrem "#{PREFIX}:busy:#{@name}", 1, Selector.uuid }
|
73
|
+
end
|
74
74
|
|
75
|
-
|
76
|
-
|
77
|
-
|
75
|
+
def probed
|
76
|
+
redis { |it| it.llen "#{PREFIX}:probed:#{@name}" }
|
77
|
+
end
|
78
78
|
|
79
|
-
|
80
|
-
|
81
|
-
|
79
|
+
def probed_processes
|
80
|
+
redis { |it| it.lrange "#{PREFIX}:probed:#{@name}", 0, -1 }
|
81
|
+
end
|
82
82
|
|
83
|
-
|
84
|
-
|
85
|
-
|
83
|
+
def pause
|
84
|
+
redis { |it| it.set "#{PREFIX}:pause:#{@name}", '1' }
|
85
|
+
end
|
86
86
|
|
87
|
-
|
88
|
-
|
89
|
-
|
87
|
+
def pause_for_ms(milliseconds)
|
88
|
+
redis { |it| it.psetex "#{PREFIX}:pause:#{@name}", milliseconds, 1 }
|
89
|
+
end
|
90
90
|
|
91
|
-
|
92
|
-
|
93
|
-
|
91
|
+
def unpause
|
92
|
+
redis { |it| it.del "#{PREFIX}:pause:#{@name}" }
|
93
|
+
end
|
94
94
|
|
95
|
-
|
96
|
-
|
97
|
-
|
95
|
+
def paused?
|
96
|
+
redis { |it| it.get "#{PREFIX}:pause:#{@name}" } == '1'
|
97
|
+
end
|
98
98
|
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
end
|
99
|
+
def block
|
100
|
+
redis { |it| it.set "#{PREFIX}:block:#{@name}", '1' }
|
101
|
+
end
|
103
102
|
|
104
|
-
|
105
|
-
|
106
|
-
end
|
103
|
+
def block_except(*queues)
|
104
|
+
raise ArgumentError if queues.empty?
|
107
105
|
|
108
|
-
|
109
|
-
|
110
|
-
end
|
106
|
+
redis { |it| it.set "#{PREFIX}:block:#{@name}", queues.join(',') }
|
107
|
+
end
|
111
108
|
|
112
|
-
|
113
|
-
|
114
|
-
%w(block busy limit pause probed process_limit).each do |key|
|
115
|
-
it.del "#{PREFIX}:#{key}:#@name"
|
109
|
+
def unblock
|
110
|
+
redis { |it| it.del "#{PREFIX}:block:#{@name}" }
|
116
111
|
end
|
117
|
-
end
|
118
|
-
end
|
119
112
|
|
120
|
-
|
121
|
-
|
122
|
-
|
113
|
+
def blocking?
|
114
|
+
redis { |it| it.get "#{PREFIX}:block:#{@name}" } == '1'
|
115
|
+
end
|
123
116
|
|
124
|
-
|
125
|
-
|
126
|
-
|
117
|
+
def clear_limits
|
118
|
+
redis do |it|
|
119
|
+
%w[block busy limit pause probed process_limit].each do |key|
|
120
|
+
it.del "#{PREFIX}:#{key}:#{@name}"
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
127
124
|
|
128
|
-
|
129
|
-
|
130
|
-
|
125
|
+
def increase_local_busy
|
126
|
+
@lock.synchronize { @local_busy += 1 }
|
127
|
+
end
|
128
|
+
|
129
|
+
def decrease_local_busy
|
130
|
+
@lock.synchronize { @local_busy -= 1 }
|
131
|
+
end
|
132
|
+
|
133
|
+
def local_busy?
|
134
|
+
@local_busy.positive?
|
135
|
+
end
|
131
136
|
|
132
|
-
|
133
|
-
|
137
|
+
def explain
|
138
|
+
<<-INFO.gsub(/^ {8}/, '')
|
134
139
|
Current sidekiq process: #{Selector.uuid}
|
135
140
|
|
136
141
|
All processes:
|
@@ -153,31 +158,33 @@ module Sidekiq::LimitFetch::Global
|
|
153
158
|
|
154
159
|
Blocking:
|
155
160
|
#{blocking?}
|
156
|
-
|
157
|
-
|
161
|
+
INFO
|
162
|
+
end
|
158
163
|
|
159
|
-
|
160
|
-
|
161
|
-
|
162
|
-
|
163
|
-
|
164
|
-
|
164
|
+
def remove_locks_except!(processes)
|
165
|
+
locked_processes = probed_processes.uniq
|
166
|
+
(locked_processes - processes).each do |dead_process|
|
167
|
+
remove_lock! dead_process
|
168
|
+
end
|
169
|
+
end
|
165
170
|
|
166
|
-
|
167
|
-
|
168
|
-
|
169
|
-
|
170
|
-
|
171
|
-
|
171
|
+
def remove_lock!(process)
|
172
|
+
redis do |it|
|
173
|
+
it.lrem "#{PREFIX}:probed:#{@name}", 0, process
|
174
|
+
it.lrem "#{PREFIX}:busy:#{@name}", 0, process
|
175
|
+
end
|
176
|
+
end
|
172
177
|
|
173
|
-
|
178
|
+
private
|
174
179
|
|
175
|
-
|
176
|
-
|
177
|
-
|
180
|
+
def redis(&block)
|
181
|
+
Sidekiq.redis(&block)
|
182
|
+
end
|
178
183
|
|
179
|
-
|
180
|
-
|
184
|
+
def namespace
|
185
|
+
Sidekiq::LimitFetch::Queues.namespace
|
186
|
+
end
|
187
|
+
end
|
181
188
|
end
|
182
189
|
end
|
183
190
|
end
|
@@ -1,23 +1,29 @@
|
|
1
|
-
|
2
|
-
def self.extended(klass)
|
3
|
-
klass.instance_variable_set :@instances, {}
|
4
|
-
end
|
1
|
+
# frozen_string_literal: true
|
5
2
|
|
6
|
-
|
7
|
-
|
8
|
-
|
3
|
+
module Sidekiq
|
4
|
+
module LimitFetch
|
5
|
+
module Instances
|
6
|
+
def self.extended(klass)
|
7
|
+
klass.instance_variable_set :@instances, {}
|
8
|
+
end
|
9
9
|
|
10
|
-
|
10
|
+
def new(*args)
|
11
|
+
@instances[args] ||= super
|
12
|
+
end
|
11
13
|
|
12
|
-
|
13
|
-
@instances.values
|
14
|
-
end
|
14
|
+
alias [] new
|
15
15
|
|
16
|
-
|
17
|
-
|
18
|
-
|
16
|
+
def instances
|
17
|
+
@instances.values
|
18
|
+
end
|
19
|
+
|
20
|
+
def reset_instances!
|
21
|
+
@instances = {}
|
22
|
+
end
|
19
23
|
|
20
|
-
|
21
|
-
|
24
|
+
def delete_instance(name)
|
25
|
+
@instances.delete [name]
|
26
|
+
end
|
27
|
+
end
|
22
28
|
end
|
23
29
|
end
|