process_balancer 1.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (23) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +0 -0
  3. data/Gemfile +8 -0
  4. data/LICENSE.txt +21 -0
  5. data/README.adoc +90 -0
  6. data/Rakefile +8 -0
  7. data/TODO.adoc +1 -0
  8. data/exe/process_balancer +17 -0
  9. data/lib/process_balancer.rb +138 -0
  10. data/lib/process_balancer/base.rb +80 -0
  11. data/lib/process_balancer/cli.rb +257 -0
  12. data/lib/process_balancer/lock/advisory_lock.rb +21 -0
  13. data/lib/process_balancer/lock/simple_redis.rb +79 -0
  14. data/lib/process_balancer/manager.rb +185 -0
  15. data/lib/process_balancer/private/cancellation.rb +116 -0
  16. data/lib/process_balancer/rails.rb +29 -0
  17. data/lib/process_balancer/redis_connection.rb +56 -0
  18. data/lib/process_balancer/util.rb +37 -0
  19. data/lib/process_balancer/version.rb +5 -0
  20. data/lib/process_balancer/watcher.rb +113 -0
  21. data/lib/process_balancer/worker.rb +76 -0
  22. data/process_balancer.gemspec +45 -0
  23. metadata +273 -0
data/lib/process_balancer/util.rb ADDED
@@ -0,0 +1,37 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'English'
4
+
5
+ module ProcessBalancer
6
+ module Util # :nodoc:
7
+ def logger
8
+ ProcessBalancer.logger
9
+ end
10
+
11
+ def hostname
12
+ ProcessBalancer.hostname
13
+ end
14
+
15
+ def identity
16
+ ProcessBalancer.identity
17
+ end
18
+
19
+ def redis(&block)
20
+ ProcessBalancer.redis(&block)
21
+ end
22
+
23
+ def start_thread(name, &block)
24
+ Thread.new do
25
+ Thread.current.name = name
26
+ watchdog(&block)
27
+ end
28
+ end
29
+
30
+ def watchdog
31
+ yield
32
+ rescue Exception => e # rubocop: disable Lint/RescueException
33
+ logger.error("#{Thread.current.name} :: #{e.message}")
34
+ raise e
35
+ end
36
+ end
37
+ end
data/lib/process_balancer/version.rb ADDED
@@ -0,0 +1,5 @@
1
+ # frozen_string_literal: true
2
+
3
+ module ProcessBalancer
4
+ VERSION = '1.0.0'
5
+ end
data/lib/process_balancer/watcher.rb ADDED
@@ -0,0 +1,113 @@
1
+ # frozen_string_literal: true
2
+
3
+ require_relative 'worker'
4
+ require_relative 'util'
5
+
6
+ module ProcessBalancer
7
+ class Watcher # :nodoc:
8
+ include Util
9
+
10
+ attr_reader :job_config, :stats
11
+
12
+ def initialize(pool, job_config)
13
+ @pool = pool
14
+ @job_config = job_config
15
+ @running = {}
16
+ @stopping = []
17
+ @stats = {}
18
+ @lock = Mutex.new
19
+ end
20
+
21
+ # called when the worker index has changed
22
+ def update_worker_config(process_index, process_count, job_count)
23
+ keep_set = process_count.zero? ? [] : (0...job_count).select { |i| (i % process_count) == process_index }
24
+
25
+ with_lock do
26
+ check_workers
27
+
28
+ running_set = @running.keys
29
+ create_set = keep_set - running_set
30
+ stop_set = running_set - keep_set
31
+
32
+ create_set.each do |worker_id|
33
+ start_worker(worker_id)
34
+ end
35
+
36
+ stop_set.each do |worker_id|
37
+ stop_worker(worker_id)
38
+ end
39
+
40
+ update_stats
41
+ end
42
+ end
43
+
44
+ private
45
+
46
+ def with_lock(&block)
47
+ @lock.synchronize(&block)
48
+ end
49
+
50
+ def start_worker(worker_id)
51
+ raise 'Not called within synchronize block' unless @lock.owned?
52
+
53
+ logger.info "Starting worker #{job_id} @ #{worker_id}"
54
+ @running[worker_id] = Worker.new(@pool, worker_id, job_config)
55
+ end
56
+
57
+ def stop_worker(worker_id, reason: '')
58
+ raise 'Not called within synchronize block' unless @lock.owned?
59
+
60
+ logger.info "Stopping worker #{job_id} @ #{worker_id} :: #{reason}"
61
+
62
+ worker = @running.delete(worker_id)
63
+ if worker
64
+ @stopping << worker
65
+ worker.stop
66
+ end
67
+ end
68
+
69
+ def check_workers
70
+ raise 'Not called within synchronize block' unless @lock.owned?
71
+
72
+ @running.each do |k, v|
73
+ if v.running?
74
+ # TODO: build up status info
75
+ else
76
+ stop_worker(k, reason: v.reason)
77
+ end
78
+ end
79
+
80
+ @stopping.delete_if do |e|
81
+ if e.stopped?
82
+ logger.debug("Reaping worker #{job_id} @ #{e.worker_index} :: #{e.reason}")
83
+ true
84
+ end
85
+ end
86
+ end
87
+
88
+ def update_stats
89
+ stats = {
90
+ running: [],
91
+ stopping: [],
92
+ }
93
+
94
+ @running.each_value do |v|
95
+ stats[:running] << {
96
+ id: v.worker_index,
97
+ }
98
+ end
99
+
100
+ @stopping.each do |v|
101
+ stats[:stopping] << {
102
+ id: v.worker_index,
103
+ }
104
+ end
105
+
106
+ @stats = stats
107
+ end
108
+
109
+ def job_id
110
+ job_config[:id]
111
+ end
112
+ end
113
+ end
data/lib/process_balancer/worker.rb ADDED
@@ -0,0 +1,76 @@
1
+ # frozen_string_literal: true
2
+
3
+ require 'concurrent/options'
4
+ # require 'concurrent/future'
5
+ require 'concurrent/promises'
6
+ require_relative 'private/cancellation'
7
+
8
+ module ProcessBalancer
9
+ class Worker # :nodoc:
10
+ attr_reader :worker_index
11
+
12
+ def initialize(pool, worker_index, job_options)
13
+ @pool = pool
14
+ @worker_index = worker_index
15
+ @job_options = job_options
16
+ cancellation, @origin = Private::Cancellation.new
17
+ @reloader = ProcessBalancer.options[:reloader]
18
+ @future = Concurrent::Promises.future_on(@pool, cancellation, &method(:runner))
19
+ end
20
+
21
+ def running?
22
+ !@origin.resolved? && !@future.resolved?
23
+ end
24
+
25
+ def stopped?
26
+ @future.resolved?
27
+ end
28
+
29
+ def stop
30
+ @origin.resolve
31
+ true
32
+ end
33
+
34
+ def reason
35
+ @future.rejected? ? @future.reason : ''
36
+ end
37
+
38
+ private
39
+
40
+ def constantize(str)
41
+ return Object.const_get(str) unless str.include?('::')
42
+
43
+ names = str.split('::')
44
+ names.shift if names.empty? || names.first.empty?
45
+
46
+ names.inject(Object) do |constant, name|
47
+ constant.const_get(name, false)
48
+ end
49
+ end
50
+
51
+ def runner(cancellation)
52
+ Thread.current.name = "Worker: #{@job_options[:id]} @ #{@worker_index}"
53
+ @reloader.call do
54
+ klass = constantize(@job_options[:class])
55
+ job = klass.new(worker_index, @job_options)
56
+
57
+ loop do
58
+ cancellation.check!
59
+
60
+ operation, *args = job.perform
61
+
62
+ case operation
63
+ when :abort
64
+ ProcessBalancer.logger.debug("Abort worker #{@job_options[:id]} @ #{@worker_index}")
65
+ cancellation.origin.resolve
66
+ when :sleep
67
+ sleep args[0]
68
+ end
69
+ end
70
+ end
71
+ rescue Concurrent::CancelledOperationError
72
+ # happy path where we finish because we were cancelled
73
+ :cancelled
74
+ end
75
+ end
76
+ end
data/process_balancer.gemspec ADDED
@@ -0,0 +1,45 @@
1
+ # frozen_string_literal: true
2
+
3
+ lib = File.expand_path('lib', __dir__)
4
+ $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
5
+ require 'process_balancer/version'
6
+
7
+ Gem::Specification.new do |spec|
8
+ spec.name = 'process_balancer'
9
+ spec.version = ProcessBalancer::VERSION
10
+ spec.authors = ['Edward Rudd']
11
+ spec.email = ['edward@hubstaff.com']
12
+
13
+ spec.summary = 'A self-balancing long-running job runner'
14
+ spec.description = 'A self-balancing long-running job runner'
15
+ spec.homepage = 'http://github.com/'
16
+ spec.license = 'LGPLv3'
17
+
18
+ # Specify which files should be added to the gem when it is released.
19
+ # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
20
+ spec.files = Dir.chdir(File.expand_path(__dir__)) do
21
+ `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(spec|bin)/}) || f[0] == '.' }
22
+ end
23
+
24
+ spec.bindir = 'exe'
25
+ spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
26
+ spec.require_paths = ['lib']
27
+
28
+ spec.required_ruby_version = '>= 2.5.0'
29
+
30
+ spec.add_development_dependency 'bundler', '~> 1.17'
31
+ spec.add_development_dependency 'climate_control', '~> 0.2'
32
+ spec.add_development_dependency 'fakefs', '~> 1.2.2'
33
+ spec.add_development_dependency 'mock_redis', '~> 0.23'
34
+ spec.add_development_dependency 'rake', '~> 11.0'
35
+ spec.add_development_dependency 'redis-namespace', '~> 1.7'
36
+ spec.add_development_dependency 'rspec', '~> 3.0'
37
+ spec.add_development_dependency 'rspec_junit_formatter', '~> 0.4'
38
+ spec.add_development_dependency 'rubocop', '~> 0.83.0'
39
+ spec.add_development_dependency 'rubocop-rspec', '~> 1.39.0'
40
+ spec.add_development_dependency 'simplecov', '~> 0.12'
41
+
42
+ spec.add_dependency 'concurrent-ruby', '~> 1.1'
43
+ spec.add_dependency 'connection_pool', '~> 2.2', '>= 2.2.2'
44
+ spec.add_dependency 'redis', '>= 3.3.5', '< 5'
45
+ end
metadata ADDED
@@ -0,0 +1,273 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: process_balancer
3
+ version: !ruby/object:Gem::Version
4
+ version: 1.0.0
5
+ platform: ruby
6
+ authors:
7
+ - Edward Rudd
8
+ autorequire:
9
+ bindir: exe
10
+ cert_chain: []
11
+ date: 2020-06-03 00:00:00.000000000 Z
12
+ dependencies:
13
+ - !ruby/object:Gem::Dependency
14
+ name: bundler
15
+ requirement: !ruby/object:Gem::Requirement
16
+ requirements:
17
+ - - "~>"
18
+ - !ruby/object:Gem::Version
19
+ version: '1.17'
20
+ type: :development
21
+ prerelease: false
22
+ version_requirements: !ruby/object:Gem::Requirement
23
+ requirements:
24
+ - - "~>"
25
+ - !ruby/object:Gem::Version
26
+ version: '1.17'
27
+ - !ruby/object:Gem::Dependency
28
+ name: climate_control
29
+ requirement: !ruby/object:Gem::Requirement
30
+ requirements:
31
+ - - "~>"
32
+ - !ruby/object:Gem::Version
33
+ version: '0.2'
34
+ type: :development
35
+ prerelease: false
36
+ version_requirements: !ruby/object:Gem::Requirement
37
+ requirements:
38
+ - - "~>"
39
+ - !ruby/object:Gem::Version
40
+ version: '0.2'
41
+ - !ruby/object:Gem::Dependency
42
+ name: fakefs
43
+ requirement: !ruby/object:Gem::Requirement
44
+ requirements:
45
+ - - "~>"
46
+ - !ruby/object:Gem::Version
47
+ version: 1.2.2
48
+ type: :development
49
+ prerelease: false
50
+ version_requirements: !ruby/object:Gem::Requirement
51
+ requirements:
52
+ - - "~>"
53
+ - !ruby/object:Gem::Version
54
+ version: 1.2.2
55
+ - !ruby/object:Gem::Dependency
56
+ name: mock_redis
57
+ requirement: !ruby/object:Gem::Requirement
58
+ requirements:
59
+ - - "~>"
60
+ - !ruby/object:Gem::Version
61
+ version: '0.23'
62
+ type: :development
63
+ prerelease: false
64
+ version_requirements: !ruby/object:Gem::Requirement
65
+ requirements:
66
+ - - "~>"
67
+ - !ruby/object:Gem::Version
68
+ version: '0.23'
69
+ - !ruby/object:Gem::Dependency
70
+ name: rake
71
+ requirement: !ruby/object:Gem::Requirement
72
+ requirements:
73
+ - - "~>"
74
+ - !ruby/object:Gem::Version
75
+ version: '11.0'
76
+ type: :development
77
+ prerelease: false
78
+ version_requirements: !ruby/object:Gem::Requirement
79
+ requirements:
80
+ - - "~>"
81
+ - !ruby/object:Gem::Version
82
+ version: '11.0'
83
+ - !ruby/object:Gem::Dependency
84
+ name: redis-namespace
85
+ requirement: !ruby/object:Gem::Requirement
86
+ requirements:
87
+ - - "~>"
88
+ - !ruby/object:Gem::Version
89
+ version: '1.7'
90
+ type: :development
91
+ prerelease: false
92
+ version_requirements: !ruby/object:Gem::Requirement
93
+ requirements:
94
+ - - "~>"
95
+ - !ruby/object:Gem::Version
96
+ version: '1.7'
97
+ - !ruby/object:Gem::Dependency
98
+ name: rspec
99
+ requirement: !ruby/object:Gem::Requirement
100
+ requirements:
101
+ - - "~>"
102
+ - !ruby/object:Gem::Version
103
+ version: '3.0'
104
+ type: :development
105
+ prerelease: false
106
+ version_requirements: !ruby/object:Gem::Requirement
107
+ requirements:
108
+ - - "~>"
109
+ - !ruby/object:Gem::Version
110
+ version: '3.0'
111
+ - !ruby/object:Gem::Dependency
112
+ name: rspec_junit_formatter
113
+ requirement: !ruby/object:Gem::Requirement
114
+ requirements:
115
+ - - "~>"
116
+ - !ruby/object:Gem::Version
117
+ version: '0.4'
118
+ type: :development
119
+ prerelease: false
120
+ version_requirements: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - "~>"
123
+ - !ruby/object:Gem::Version
124
+ version: '0.4'
125
+ - !ruby/object:Gem::Dependency
126
+ name: rubocop
127
+ requirement: !ruby/object:Gem::Requirement
128
+ requirements:
129
+ - - "~>"
130
+ - !ruby/object:Gem::Version
131
+ version: 0.83.0
132
+ type: :development
133
+ prerelease: false
134
+ version_requirements: !ruby/object:Gem::Requirement
135
+ requirements:
136
+ - - "~>"
137
+ - !ruby/object:Gem::Version
138
+ version: 0.83.0
139
+ - !ruby/object:Gem::Dependency
140
+ name: rubocop-rspec
141
+ requirement: !ruby/object:Gem::Requirement
142
+ requirements:
143
+ - - "~>"
144
+ - !ruby/object:Gem::Version
145
+ version: 1.39.0
146
+ type: :development
147
+ prerelease: false
148
+ version_requirements: !ruby/object:Gem::Requirement
149
+ requirements:
150
+ - - "~>"
151
+ - !ruby/object:Gem::Version
152
+ version: 1.39.0
153
+ - !ruby/object:Gem::Dependency
154
+ name: simplecov
155
+ requirement: !ruby/object:Gem::Requirement
156
+ requirements:
157
+ - - "~>"
158
+ - !ruby/object:Gem::Version
159
+ version: '0.12'
160
+ type: :development
161
+ prerelease: false
162
+ version_requirements: !ruby/object:Gem::Requirement
163
+ requirements:
164
+ - - "~>"
165
+ - !ruby/object:Gem::Version
166
+ version: '0.12'
167
+ - !ruby/object:Gem::Dependency
168
+ name: concurrent-ruby
169
+ requirement: !ruby/object:Gem::Requirement
170
+ requirements:
171
+ - - "~>"
172
+ - !ruby/object:Gem::Version
173
+ version: '1.1'
174
+ type: :runtime
175
+ prerelease: false
176
+ version_requirements: !ruby/object:Gem::Requirement
177
+ requirements:
178
+ - - "~>"
179
+ - !ruby/object:Gem::Version
180
+ version: '1.1'
181
+ - !ruby/object:Gem::Dependency
182
+ name: connection_pool
183
+ requirement: !ruby/object:Gem::Requirement
184
+ requirements:
185
+ - - "~>"
186
+ - !ruby/object:Gem::Version
187
+ version: '2.2'
188
+ - - ">="
189
+ - !ruby/object:Gem::Version
190
+ version: 2.2.2
191
+ type: :runtime
192
+ prerelease: false
193
+ version_requirements: !ruby/object:Gem::Requirement
194
+ requirements:
195
+ - - "~>"
196
+ - !ruby/object:Gem::Version
197
+ version: '2.2'
198
+ - - ">="
199
+ - !ruby/object:Gem::Version
200
+ version: 2.2.2
201
+ - !ruby/object:Gem::Dependency
202
+ name: redis
203
+ requirement: !ruby/object:Gem::Requirement
204
+ requirements:
205
+ - - ">="
206
+ - !ruby/object:Gem::Version
207
+ version: 3.3.5
208
+ - - "<"
209
+ - !ruby/object:Gem::Version
210
+ version: '5'
211
+ type: :runtime
212
+ prerelease: false
213
+ version_requirements: !ruby/object:Gem::Requirement
214
+ requirements:
215
+ - - ">="
216
+ - !ruby/object:Gem::Version
217
+ version: 3.3.5
218
+ - - "<"
219
+ - !ruby/object:Gem::Version
220
+ version: '5'
221
+ description: A self-balancing long-running job runner
222
+ email:
223
+ - edward@hubstaff.com
224
+ executables:
225
+ - process_balancer
226
+ extensions: []
227
+ extra_rdoc_files: []
228
+ files:
229
+ - CHANGELOG.md
230
+ - Gemfile
231
+ - LICENSE.txt
232
+ - README.adoc
233
+ - Rakefile
234
+ - TODO.adoc
235
+ - exe/process_balancer
236
+ - lib/process_balancer.rb
237
+ - lib/process_balancer/base.rb
238
+ - lib/process_balancer/cli.rb
239
+ - lib/process_balancer/lock/advisory_lock.rb
240
+ - lib/process_balancer/lock/simple_redis.rb
241
+ - lib/process_balancer/manager.rb
242
+ - lib/process_balancer/private/cancellation.rb
243
+ - lib/process_balancer/rails.rb
244
+ - lib/process_balancer/redis_connection.rb
245
+ - lib/process_balancer/util.rb
246
+ - lib/process_balancer/version.rb
247
+ - lib/process_balancer/watcher.rb
248
+ - lib/process_balancer/worker.rb
249
+ - process_balancer.gemspec
250
+ homepage: http://github.com/
251
+ licenses:
252
+ - LGPLv3
253
+ metadata: {}
254
+ post_install_message:
255
+ rdoc_options: []
256
+ require_paths:
257
+ - lib
258
+ required_ruby_version: !ruby/object:Gem::Requirement
259
+ requirements:
260
+ - - ">="
261
+ - !ruby/object:Gem::Version
262
+ version: 2.5.0
263
+ required_rubygems_version: !ruby/object:Gem::Requirement
264
+ requirements:
265
+ - - ">="
266
+ - !ruby/object:Gem::Version
267
+ version: '0'
268
+ requirements: []
269
+ rubygems_version: 3.0.6
270
+ signing_key:
271
+ specification_version: 4
272
+ summary: A self-balancing long-running job runner
273
+ test_files: []