worker_killer 1.0.4.189871 → 1.0.5.213977
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +37 -3
- data/lib/worker_killer/killer/puma.rb +62 -0
- data/lib/worker_killer/killer.rb +1 -0
- data/lib/worker_killer/puma/plugin/worker_killer.rb +104 -0
- data/lib/worker_killer/puma_plugin.rb +58 -0
- data/lib/worker_killer/version.rb +1 -1
- data/spec/killer/puma_spec.rb +33 -0
- data/spec/killer_spec.rb +1 -1
- metadata +7 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2ac9af870733f50ff006c5a31d9c3030342d76ff3208b8e4193a2407ad8f264c
|
4
|
+
data.tar.gz: c222a1cada623f95f8668e61d15e4f32fa986fdf799572e1937365919bc651d4
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 88918e5de623c64691e96fbfc4f34b6eb19669e3f370be011eed996ef6e9a13aee61997b66adfae929e39cb685396557c3802d9a00dcb0ebef0d54e5cf1193fb
|
7
|
+
data.tar.gz: bdb4e2fafb1da70554c3b73284538695bfd918b1b516a9a24f477b02056d721b69745f4ecfe6bfde044fb90815fa55f7178c08c33d9cfd283420754d0175ee69
|
data/README.md
CHANGED
@@ -17,6 +17,7 @@ Features:
|
|
17
17
|
|
18
18
|
- generic middleware implementation
|
19
19
|
- Phusion Passenger support(through `passenger-config detach-process <PID>`)
|
20
|
+
- Puma phased-restart support(through `pumactl phased-restart`)
|
20
21
|
- DelayedJob support
|
21
22
|
- custom reaction hook
|
22
23
|
|
@@ -42,14 +43,14 @@ Add these lines to your `config.ru` or `application.rb`. (These lines should be
|
|
42
43
|
|
43
44
|
# Max requests per worker
|
44
45
|
middleware.insert_before(
|
45
|
-
Rack::
|
46
|
+
Rack::Runtime,
|
46
47
|
WorkerKiller::Middleware::RequestsLimiter, killer: killer, min: 3072, max: 4096
|
47
48
|
)
|
48
49
|
|
49
50
|
# Max memory size (RSS) per worker
|
50
51
|
middleware.insert_before(
|
51
|
-
Rack::
|
52
|
-
WorkerKiller::Middleware::OOMLimiter, killer: killer, min:
|
52
|
+
Rack::Runtime,
|
53
|
+
WorkerKiller::Middleware::OOMLimiter, killer: killer, min: nil, max: 0.5, check_cycle: 16
|
53
54
|
)
|
54
55
|
```
|
55
56
|
|
@@ -74,6 +75,39 @@ Add these lines to your `initializers/delayed_job.rb` or `application.rb`.
|
|
74
75
|
end
|
75
76
|
```
|
76
77
|
|
78
|
+
## Puma Web-server
|
79
|
+
|
80
|
+
Add these lines to your `puma.rb` AND `application.rb`.
|
81
|
+
|
82
|
+
```ruby
|
83
|
+
# puma.rb
|
84
|
+
|
85
|
+
on_worker_boot do |*args|
|
86
|
+
require 'worker_killer/killer'
|
87
|
+
require 'active_support/notifications'
|
88
|
+
|
89
|
+
::ActiveSupport::Notifications.subscribe 'initialize.custom.rails' do |*eargs|
|
90
|
+
event = ActiveSupport::Notifications::Event.new(*eargs)
|
91
|
+
config = event.payload[:config]
|
92
|
+
|
93
|
+
killer = ::WorkerKiller::Killer::Puma.new
|
94
|
+
config.middleware.insert_before(
|
95
|
+
::Rack::Runtime,
|
96
|
+
::WorkerKiller::Middleware::RequestsLimiter, killer: killer, min: 3072, max: 4096
|
97
|
+
)
|
98
|
+
|
99
|
+
config.middleware.insert_before(
|
100
|
+
::Rack::Runtime,
|
101
|
+
::WorkerKiller::Middleware::OOMLimiter, killer: killer, min: nil, max: 0.5, verbose: true, check_cycle: 16
|
102
|
+
)
|
103
|
+
end
|
104
|
+
end
|
105
|
+
|
106
|
+
# application.rb
|
107
|
+
|
108
|
+
ActiveSupport::Notifications.instrument 'initialize.custom.rails', config: config
|
109
|
+
```
|
110
|
+
|
77
111
|
This gem provides two modules: WorkerKiller::CountLimiter and WorkerKiller::MemoryLimiter, some Rack integration and DelayedJob plugin.
|
78
112
|
|
79
113
|
### WorkerKiller::Middleware::RequestsLimiter and WorkerKiller::DelayedJobPlugin::JobsLimiter
|
@@ -0,0 +1,62 @@
|
|
1
|
+
require 'socket'
|
2
|
+
|
3
|
+
module ::WorkerKiller
|
4
|
+
module Killer
|
5
|
+
class Puma < ::WorkerKiller::Killer::Base
|
6
|
+
|
7
|
+
attr_accessor :type, :plugin_path, :num
|
8
|
+
|
9
|
+
def initialize(type: :phased, path: nil, num: nil, **kwargs)
|
10
|
+
super(**kwargs)
|
11
|
+
@type = type
|
12
|
+
@plugin_path = path
|
13
|
+
@num = num
|
14
|
+
end
|
15
|
+
|
16
|
+
def do_kill(sig, pid, alive_sec, **params)
|
17
|
+
if @type == :phased
|
18
|
+
do_phased_kill(sig, pid, alive_sec, **params)
|
19
|
+
elsif @type == :plugin
|
20
|
+
do_plugin_kill(sig, pid, alive_sec, **params)
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
def do_phased_kill(sig, pid, alive_sec, **_params)
|
25
|
+
cmd = 'pumactl phased-restart'
|
26
|
+
|
27
|
+
if sig == :KILL
|
28
|
+
logger.error "#{self} force to kill self (pid: #{pid}) alive: #{alive_sec} sec (trial #{kill_attempts})"
|
29
|
+
Process.kill sig, pid
|
30
|
+
return
|
31
|
+
end
|
32
|
+
|
33
|
+
return if @already_detached
|
34
|
+
|
35
|
+
logger.warn "#{self} run #{cmd.inspect} (pid: #{pid}) alive: #{alive_sec} sec (trial #{kill_attempts})"
|
36
|
+
@already_detached = true
|
37
|
+
|
38
|
+
Thread.new(cmd) do |command|
|
39
|
+
unless Kernel.system(command)
|
40
|
+
logger.warn "#{self} run #{command.inspect} failed: #{$?.inspect}"
|
41
|
+
end
|
42
|
+
end
|
43
|
+
end
|
44
|
+
|
45
|
+
def do_plugin_kill(sig, pid, alive_sec, **_params)
|
46
|
+
if sig == :KILL
|
47
|
+
logger.error "#{self} force to kill self (pid: #{pid}) alive: #{alive_sec} sec (trial #{kill_attempts})"
|
48
|
+
Process.kill sig, pid
|
49
|
+
return
|
50
|
+
end
|
51
|
+
|
52
|
+
logger.warn "#{self} send #{num} to Puma Plugin (pid: #{pid}) alive: #{alive_sec} sec (trial #{kill_attempts})"
|
53
|
+
|
54
|
+
Socket.unix(plugin_path) do |sock|
|
55
|
+
sock.puts num.to_s
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
62
|
+
|
data/lib/worker_killer/killer.rb
CHANGED
@@ -0,0 +1,104 @@
|
|
1
|
+
require 'socket'
|
2
|
+
require 'puma/plugin'
|
3
|
+
require 'active_support/notifications'
|
4
|
+
|
5
|
+
require 'worker_killer/killer'
|
6
|
+
|
7
|
+
|
8
|
+
|
9
|
+
|
10
|
+
module WorkerKiller
|
11
|
+
module Puma
|
12
|
+
module Plugin
|
13
|
+
# Puma requires such plugin name and path :(
|
14
|
+
class WorkerKiller
|
15
|
+
|
16
|
+
def initialize(launcher, path)
|
17
|
+
@runner = launcher.instance_variable_get('@runner')
|
18
|
+
|
19
|
+
@thread = Thread.new do
|
20
|
+
Socket.unix_server_loop(path) do |sock, _client_addrinfo|
|
21
|
+
if (line = sock.gets)
|
22
|
+
worker_num = Integer(line.strip)
|
23
|
+
if (worker = find_worker(worker_num))
|
24
|
+
log "Killing worker #{worker_num}"
|
25
|
+
worker.term!
|
26
|
+
end
|
27
|
+
end
|
28
|
+
rescue StandardError => e
|
29
|
+
log("Exception: #{e.inspect}")
|
30
|
+
ensure
|
31
|
+
sock.close
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def find_worker(worker_num)
|
37
|
+
worker = @runner.worker_at(worker_num)
|
38
|
+
unless worker
|
39
|
+
log "Unknown worker index: #{worker_num.inspect}. Skipping."
|
40
|
+
return nil
|
41
|
+
end
|
42
|
+
|
43
|
+
unless worker.booted?
|
44
|
+
log "Worker #{worker_num.inspect} is not booted yet. Skipping."
|
45
|
+
return nil
|
46
|
+
end
|
47
|
+
|
48
|
+
if worker.term?
|
49
|
+
log "Worker #{worker_num.inspect} already terminating. Skipping."
|
50
|
+
return nil
|
51
|
+
end
|
52
|
+
|
53
|
+
worker
|
54
|
+
end
|
55
|
+
|
56
|
+
def log(msg)
|
57
|
+
warn("#{self.class}: #{msg}")
|
58
|
+
end
|
59
|
+
|
60
|
+
end
|
61
|
+
end
|
62
|
+
end
|
63
|
+
end
|
64
|
+
|
65
|
+
Puma::Plugin.create do
|
66
|
+
attr_reader :path
|
67
|
+
|
68
|
+
def config(c)
|
69
|
+
c.on_worker_boot do |num|
|
70
|
+
|
71
|
+
if @killer
|
72
|
+
puts "ON OTHER WORKER BOOT: #{num}"
|
73
|
+
@killer.num = num
|
74
|
+
else
|
75
|
+
puts "ON MAIN WORKER BOOT: #{num}"
|
76
|
+
::ActiveSupport::Notifications.subscribe 'worker_killer.initialize' do |*eargs|
|
77
|
+
@killer = ::WorkerKiller::Killer::Puma.new(num: num, path: path, type: :plugin)
|
78
|
+
puts "ON WORKER BOOT INIT:#{num}"
|
79
|
+
event = ActiveSupport::Notifications::Event.new(*eargs)
|
80
|
+
config = event.payload[:config]
|
81
|
+
|
82
|
+
@register_killer&.call(self, config, @killer)
|
83
|
+
end
|
84
|
+
end
|
85
|
+
end
|
86
|
+
end
|
87
|
+
|
88
|
+
def start(launcher)
|
89
|
+
@path = File.join('tmp', "puma_worker_killer_#{Process.pid}.socket")
|
90
|
+
|
91
|
+
launcher.events.on_booted do
|
92
|
+
@controller ||= WorkerKiller::Puma::Plugin::WorkerKiller.new(launcher, path)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
def configure(&block)
|
97
|
+
self.instance_eval(&block)
|
98
|
+
end
|
99
|
+
|
100
|
+
def register_killer(&block)
|
101
|
+
@register_killer = block
|
102
|
+
end
|
103
|
+
end
|
104
|
+
|
@@ -0,0 +1,58 @@
|
|
1
|
+
require 'worker_killer/memory_limiter'
|
2
|
+
require 'worker_killer/count_limiter'
|
3
|
+
require 'puma/plugin'
|
4
|
+
|
5
|
+
Puma::Plugin.create do
|
6
|
+
|
7
|
+
module WorkerKiller
|
8
|
+
class PumaPlugin
|
9
|
+
|
10
|
+
attr_reader :limiter, :killer, :reaction
|
11
|
+
|
12
|
+
def initialize(klass:, killer:, reaction: nil, **opts)
|
13
|
+
@killer = killer
|
14
|
+
|
15
|
+
@reaction = reaction || proc do |l, k, dj|
|
16
|
+
k.kill(l.started_at, dj: dj)
|
17
|
+
end
|
18
|
+
|
19
|
+
@limiter = klass.new(**opts)
|
20
|
+
@time_to_burn = false
|
21
|
+
end
|
22
|
+
|
23
|
+
def new(lifecycle = Delayed::Worker.lifecycle, *_args)
|
24
|
+
configure_lifecycle(lifecycle)
|
25
|
+
end
|
26
|
+
|
27
|
+
def configure_lifecycle(lifecycle)
|
28
|
+
# Count condition after every job
|
29
|
+
lifecycle.after(:perform) do |worker, *_args|
|
30
|
+
@time_to_burn ||= limiter.check
|
31
|
+
end
|
32
|
+
|
33
|
+
# Stop execution only after whole loop completed
|
34
|
+
lifecycle.after(:loop) do |worker, *_args|
|
35
|
+
@time_to_burn ||= limiter.check
|
36
|
+
reaction.call(limiter, killer, worker) if @time_to_burn
|
37
|
+
end
|
38
|
+
end
|
39
|
+
|
40
|
+
class JobsLimiter < ::WorkerKiller::PumaPlugin
|
41
|
+
|
42
|
+
def initialize(**opts)
|
43
|
+
super(klass: ::WorkerKiller::CountLimiter, **opts)
|
44
|
+
end
|
45
|
+
|
46
|
+
end
|
47
|
+
|
48
|
+
class OOMLimiter < ::WorkerKiller::PumaPlugin
|
49
|
+
|
50
|
+
def initialize(**opts)
|
51
|
+
super(klass: ::WorkerKiller::MemoryLimiter, **opts)
|
52
|
+
end
|
53
|
+
|
54
|
+
end
|
55
|
+
|
56
|
+
end
|
57
|
+
end
|
58
|
+
|
@@ -0,0 +1,33 @@
|
|
1
|
+
RSpec.describe WorkerKiller::Killer::Puma do
|
2
|
+
let(:config) do
|
3
|
+
WorkerKiller::Configuration.new.tap do |c|
|
4
|
+
c.quit_attempts = 2
|
5
|
+
c.term_attempts = 2
|
6
|
+
end
|
7
|
+
end
|
8
|
+
|
9
|
+
let(:killer){ described_class.new }
|
10
|
+
|
11
|
+
describe '#kill' do
|
12
|
+
around do |example|
|
13
|
+
prev = WorkerKiller.configuration
|
14
|
+
WorkerKiller.configuration = config
|
15
|
+
example.run
|
16
|
+
ensure
|
17
|
+
WorkerKiller.configuration = prev
|
18
|
+
end
|
19
|
+
|
20
|
+
it 'expect right signal order' do
|
21
|
+
expect(Kernel).to receive(:system).with('pumactl phased-restart').and_return(true)
|
22
|
+
expect(Process).to receive(:kill).with(:KILL, anything).exactly(5).times
|
23
|
+
|
24
|
+
thread = killer.kill(Time.now)
|
25
|
+
thread.join
|
26
|
+
|
27
|
+
1.times { killer.kill(Time.now) } # 1 QUIT
|
28
|
+
2.times { killer.kill(Time.now) } # 1 TERM
|
29
|
+
5.times { killer.kill(Time.now) } # 5 KILL
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
data/spec/killer_spec.rb
CHANGED
@@ -19,7 +19,7 @@ RSpec.describe WorkerKiller::Killer::Base do
|
|
19
19
|
end
|
20
20
|
|
21
21
|
it 'expect right signal order' do
|
22
|
-
if RUBY_VERSION >= "
|
22
|
+
if RUBY_VERSION >= "2.7.0"
|
23
23
|
expect(killer).to receive(:do_kill).with(:QUIT, anything, anything).exactly(2).times
|
24
24
|
expect(killer).to receive(:do_kill).with(:TERM, anything, anything).exactly(2).times
|
25
25
|
expect(killer).to receive(:do_kill).with(:KILL, anything, anything).exactly(5).times
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: worker_killer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.0.
|
4
|
+
version: 1.0.5.213977
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Samoilenko Yuri
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date:
|
11
|
+
date: 2024-02-22 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: get_process_mem
|
@@ -131,15 +131,19 @@ files:
|
|
131
131
|
- lib/worker_killer/killer.rb
|
132
132
|
- lib/worker_killer/killer/delayed_job.rb
|
133
133
|
- lib/worker_killer/killer/passenger.rb
|
134
|
+
- lib/worker_killer/killer/puma.rb
|
134
135
|
- lib/worker_killer/killer/signal.rb
|
135
136
|
- lib/worker_killer/memory_limiter.rb
|
136
137
|
- lib/worker_killer/middleware.rb
|
138
|
+
- lib/worker_killer/puma/plugin/worker_killer.rb
|
139
|
+
- lib/worker_killer/puma_plugin.rb
|
137
140
|
- lib/worker_killer/version.rb
|
138
141
|
- spec/count_limiter_spec.rb
|
139
142
|
- spec/delayed_job_plugin/jobs_limiter_spec.rb
|
140
143
|
- spec/delayed_job_plugin/oom_limiter_spec.rb
|
141
144
|
- spec/killer/delayed_job_spec.rb
|
142
145
|
- spec/killer/passenger_spec.rb
|
146
|
+
- spec/killer/puma_spec.rb
|
143
147
|
- spec/killer/signal_spec.rb
|
144
148
|
- spec/killer_spec.rb
|
145
149
|
- spec/memory_limiter_spec.rb
|
@@ -177,6 +181,7 @@ test_files:
|
|
177
181
|
- spec/delayed_job_plugin/oom_limiter_spec.rb
|
178
182
|
- spec/delayed_job_plugin/jobs_limiter_spec.rb
|
179
183
|
- spec/killer/passenger_spec.rb
|
184
|
+
- spec/killer/puma_spec.rb
|
180
185
|
- spec/killer/signal_spec.rb
|
181
186
|
- spec/killer/delayed_job_spec.rb
|
182
187
|
- spec/memory_limiter_spec.rb
|