worker_killer 1.2.0.338627 → 1.2.1.338700
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/worker_killer/configuration.rb +3 -0
- data/lib/worker_killer/count_limiter.rb +12 -8
- data/lib/worker_killer/killer/delayed_job.rb +10 -14
- data/lib/worker_killer/killer/puma.rb +1 -13
- data/lib/worker_killer/killer/signal.rb +2 -2
- data/lib/worker_killer/killer.rb +10 -4
- data/lib/worker_killer/memory_limiter.rb +16 -10
- data/lib/worker_killer/middleware.rb +65 -35
- data/lib/worker_killer/puma_plugin.rb +2 -0
- data/lib/worker_killer/puma_plugin_ng.rb +14 -26
- data/lib/worker_killer/version.rb +1 -1
- data/spec/killer/delayed_job_spec.rb +7 -26
- data/spec/killer/passenger_spec.rb +6 -23
- data/spec/killer/puma_spec.rb +0 -16
- data/spec/killer/signal_spec.rb +7 -25
- data/spec/killer_spec.rb +15 -31
- data/spec/middleware_spec.rb +14 -10
- data/spec/puma_plugin_ng_spec.rb +1 -18
- metadata +2 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 986b0714653c95814fad3451e5dd4b94f7bfbd51d906862ba9162e6a43538323
|
|
4
|
+
data.tar.gz: dbd7caf55941dcd7507a5faff2e70accb5506149c5d3aee5f5cf5e87bc9b3999
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: e85bbb81be5b5d24615958f95d23c41e892b7212a1fbb5c09aa51b97887bb50dc857faef09e9e4bf686b169c395ded52868d0d8715482ca44f9897cedfb93d21
|
|
7
|
+
data.tar.gz: cb15d46503bcb63785a77c3e5cdcf602edce81fb7cd939da9d1047bdbe21686e6aad6c8f1137b15e079777f2f2179e8e8a70cd5c3ab0be379ad553a4929ab7bf
|
|
@@ -4,10 +4,13 @@ module WorkerKiller
|
|
|
4
4
|
# Methods for configuring WorkerKiller
|
|
5
5
|
class Configuration
|
|
6
6
|
|
|
7
|
+
# Attempts configuration is deprecated and unsed
|
|
7
8
|
attr_accessor :logger, :quit_attempts, :term_attempts
|
|
8
9
|
|
|
9
10
|
# Override defaults for configuration
|
|
11
|
+
# Attempts configuration is deprecated and unsed
|
|
10
12
|
def initialize(quit_attempts: 10, term_attempts: 50)
|
|
13
|
+
# Attempts configuration is deprecated and unsed
|
|
11
14
|
@quit_attempts = quit_attempts
|
|
12
15
|
@term_attempts = term_attempts
|
|
13
16
|
@logger = Logger.new(STDOUT, level: Logger::INFO, progname: 'WorkerKiller')
|
|
@@ -8,19 +8,23 @@ module WorkerKiller
|
|
|
8
8
|
@max = max
|
|
9
9
|
@limit = nil
|
|
10
10
|
@left = nil
|
|
11
|
+
|
|
12
|
+
@started_at = Time.now
|
|
13
|
+
@triggered = false
|
|
11
14
|
@verbose = verbose
|
|
12
15
|
end
|
|
13
16
|
|
|
17
|
+
def initialize_limits
|
|
18
|
+
@limit = min + WorkerKiller.randomize(max - min + 1)
|
|
19
|
+
@left = @limit
|
|
20
|
+
end
|
|
21
|
+
|
|
14
22
|
def check
|
|
15
|
-
|
|
16
|
-
if @limit.nil?
|
|
17
|
-
@limit = min + WorkerKiller.randomize(max - min + 1)
|
|
18
|
-
@left = @limit
|
|
19
|
-
end
|
|
23
|
+
return true if @triggered
|
|
20
24
|
|
|
21
|
-
|
|
25
|
+
initialize_limits if @limit.nil?
|
|
22
26
|
|
|
23
|
-
@
|
|
27
|
+
return nil if @limit < 1
|
|
24
28
|
|
|
25
29
|
if @verbose
|
|
26
30
|
logger.info "#{self.class}: worker (pid: #{Process.pid}) has #{@left} left before being limited"
|
|
@@ -28,8 +32,8 @@ module WorkerKiller
|
|
|
28
32
|
|
|
29
33
|
return false if (@left -= 1) > 0
|
|
30
34
|
|
|
35
|
+
# сработает только один раз для @left == 0
|
|
31
36
|
logger.warn "#{self.class}: worker (pid: #{Process.pid}) exceeds max number of requests (limit: #{@limit})"
|
|
32
|
-
|
|
33
37
|
true
|
|
34
38
|
end
|
|
35
39
|
|
|
@@ -2,23 +2,19 @@ module WorkerKiller
|
|
|
2
2
|
module Killer
|
|
3
3
|
class DelayedJob < ::WorkerKiller::Killer::Base
|
|
4
4
|
|
|
5
|
-
def do_kill(sig, pid, alive_sec, dj:, **
|
|
6
|
-
|
|
7
|
-
|
|
5
|
+
def do_kill(sig, pid, alive_sec, dj:, **_kwargs)
|
|
6
|
+
case sig
|
|
7
|
+
when :QUIT
|
|
8
|
+
logger.info "#{self.class}: try to stop DelayedJob due to #{sig} self (pid: #{pid}) alive: #{alive_sec} sec (kill attempts #{kill_attempts})"
|
|
9
|
+
dj.stop
|
|
10
|
+
when :TERM
|
|
11
|
+
logger.warn "#{self.class}: force to #{sig} self (pid: #{pid}) alive: #{alive_sec} sec (kill attempts #{kill_attempts})"
|
|
8
12
|
Process.kill sig, pid
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
dj.stop
|
|
13
|
-
logger.info "#{self.class}: try to stop DelayedJob due to #{sig} self (pid: #{pid}) alive: #{alive_sec} sec (trial #{kill_attempts})"
|
|
14
|
-
|
|
15
|
-
return if sig != :TERM
|
|
16
|
-
|
|
17
|
-
if @termination
|
|
18
|
-
logger.warn "#{self.class}: force to #{sig} self (pid: #{pid}) alive: #{alive_sec} sec (trial #{kill_attempts})"
|
|
13
|
+
when :KILL
|
|
14
|
+
logger.error "#{self.class}: force to #{sig} self (pid: #{pid}) alive: #{alive_sec} sec (kill attempts #{kill_attempts})"
|
|
19
15
|
Process.kill sig, pid
|
|
20
16
|
else
|
|
21
|
-
|
|
17
|
+
logger.error "#{self.class}: DO NOTHING: unknown #{sig} self (pid: #{pid}) alive: #{alive_sec} sec (kill attempts #{kill_attempts})"
|
|
22
18
|
end
|
|
23
19
|
end
|
|
24
20
|
|
|
@@ -14,7 +14,7 @@ module ::WorkerKiller
|
|
|
14
14
|
return if @worker_num.nil? # May be nil if Puma not in Cluster mode
|
|
15
15
|
return if @already_sended == sig
|
|
16
16
|
|
|
17
|
-
logger.warn "#{self.class} send [W#{worker_num}] to Puma Plugin (from pid: #{pid}) alive: #{alive_sec} sec (
|
|
17
|
+
logger.warn "#{self.class} send [W#{worker_num}] to Puma Plugin (from pid: #{pid}) alive: #{alive_sec} sec (kill attempts #{kill_attempts}) triggered by #{sig}"
|
|
18
18
|
|
|
19
19
|
@already_sended = sig
|
|
20
20
|
@puma_plugin.set_logger!(logger)
|
|
@@ -22,18 +22,6 @@ module ::WorkerKiller
|
|
|
22
22
|
@puma_plugin.request_restart_server(worker_num)
|
|
23
23
|
end
|
|
24
24
|
|
|
25
|
-
def do_inhibit(_path_info)
|
|
26
|
-
@puma_plugin.set_logger!(logger)
|
|
27
|
-
|
|
28
|
-
@puma_plugin.inhibit_restart(worker_num)
|
|
29
|
-
end
|
|
30
|
-
|
|
31
|
-
def do_release
|
|
32
|
-
@puma_plugin.set_logger!(logger)
|
|
33
|
-
|
|
34
|
-
@puma_plugin.release_restart(worker_num)
|
|
35
|
-
end
|
|
36
|
-
|
|
37
25
|
end
|
|
38
26
|
end
|
|
39
27
|
end
|
|
@@ -2,11 +2,11 @@ module WorkerKiller
|
|
|
2
2
|
module Killer
|
|
3
3
|
class Signal < ::WorkerKiller::Killer::Base
|
|
4
4
|
|
|
5
|
-
def do_kill(sig, pid, alive_sec, **
|
|
5
|
+
def do_kill(sig, pid, alive_sec, **_kwargs)
|
|
6
6
|
return if sig == @last_signal
|
|
7
7
|
|
|
8
8
|
@last_signal = sig
|
|
9
|
-
logger.warn "#{self} send SIG#{sig} (pid: #{pid}) alive: #{alive_sec} sec (
|
|
9
|
+
logger.warn "#{self} send SIG#{sig} (pid: #{pid}) alive: #{alive_sec} sec (kill attempts #{kill_attempts})"
|
|
10
10
|
Process.kill sig, pid
|
|
11
11
|
end
|
|
12
12
|
|
data/lib/worker_killer/killer.rb
CHANGED
|
@@ -8,6 +8,7 @@ module WorkerKiller
|
|
|
8
8
|
@logger = logger
|
|
9
9
|
@config = WorkerKiller.configuration
|
|
10
10
|
@kill_attempts = 0
|
|
11
|
+
@sig = nil
|
|
11
12
|
end
|
|
12
13
|
|
|
13
14
|
def kill(start_time, **params)
|
|
@@ -15,11 +16,16 @@ module WorkerKiller
|
|
|
15
16
|
|
|
16
17
|
@kill_attempts += 1
|
|
17
18
|
|
|
18
|
-
sig
|
|
19
|
-
|
|
20
|
-
|
|
19
|
+
case @sig
|
|
20
|
+
when nil
|
|
21
|
+
@sig = :QUIT
|
|
22
|
+
when :QUIT
|
|
23
|
+
@sig = :TERM
|
|
24
|
+
when :TERM
|
|
25
|
+
@sig = :KILL
|
|
26
|
+
end
|
|
21
27
|
|
|
22
|
-
do_kill(sig, Process.pid, alive_sec, **params)
|
|
28
|
+
do_kill(@sig, Process.pid, alive_sec, **params)
|
|
23
29
|
end
|
|
24
30
|
|
|
25
31
|
# :nocov:
|
|
@@ -15,28 +15,32 @@ module WorkerKiller
|
|
|
15
15
|
@max_percent = max
|
|
16
16
|
end
|
|
17
17
|
|
|
18
|
+
@started_at = Time.now
|
|
18
19
|
@check_cycle = check_cycle
|
|
19
20
|
@check_count = 0
|
|
21
|
+
|
|
22
|
+
@triggered = false
|
|
20
23
|
@verbose = verbose
|
|
21
24
|
end
|
|
22
25
|
|
|
26
|
+
def initialize_limits(rss)
|
|
27
|
+
if min.nil?
|
|
28
|
+
set_limits(rss, rss + rss * @max_percent)
|
|
29
|
+
else
|
|
30
|
+
set_limits(min, min + WorkerKiller.randomize(max - min + 1))
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
|
23
34
|
def check
|
|
24
|
-
@started_at ||= Time.now
|
|
25
35
|
@check_count += 1
|
|
26
|
-
|
|
27
|
-
|
|
28
36
|
return nil if (@check_count % @check_cycle) != 0
|
|
29
37
|
|
|
38
|
+
return true if @triggered
|
|
39
|
+
|
|
30
40
|
rss = GetProcessMem.new.bytes
|
|
31
41
|
|
|
32
42
|
# initialize relative memory limits on first check
|
|
33
|
-
if @limit.nil?
|
|
34
|
-
if min.nil?
|
|
35
|
-
set_limits(rss, rss + rss * @max_percent)
|
|
36
|
-
else
|
|
37
|
-
set_limits(min, min + WorkerKiller.randomize(max - min + 1))
|
|
38
|
-
end
|
|
39
|
-
end
|
|
43
|
+
initialize_limits(rss) if @limit.nil?
|
|
40
44
|
|
|
41
45
|
do_check(rss)
|
|
42
46
|
end
|
|
@@ -51,6 +55,8 @@ module WorkerKiller
|
|
|
51
55
|
|
|
52
56
|
return false if rss <= @limit
|
|
53
57
|
|
|
58
|
+
@triggered = true
|
|
59
|
+
|
|
54
60
|
logger.warn "#{self.class}: worker (pid: #{Process.pid}) exceeds memory limit (#{rss_mb} MB > #{@limit_mb} MB)"
|
|
55
61
|
|
|
56
62
|
true
|
|
@@ -4,7 +4,7 @@ require 'worker_killer/count_limiter'
|
|
|
4
4
|
module WorkerKiller
|
|
5
5
|
class Middleware
|
|
6
6
|
|
|
7
|
-
attr_reader :limiter, :killer, :reaction, :inhibitions
|
|
7
|
+
attr_reader :limiter, :killer, :interval, :reaction, :inhibitions
|
|
8
8
|
|
|
9
9
|
# inhibitions - список адресов, которые будут времнно блокировать перезапуск воркеров:
|
|
10
10
|
# Rails.application.config.middleware.insert_before(
|
|
@@ -12,68 +12,98 @@ module WorkerKiller
|
|
|
12
12
|
# WorkerKiller::Middleware::RequestsLimiter, killer:, min: 2, max: 3, inhibitions: ['/test']
|
|
13
13
|
# )
|
|
14
14
|
#
|
|
15
|
-
def initialize(app, killer:, klass:, inhibitions: [], reaction: nil, **opts)
|
|
15
|
+
def initialize(app, killer:, klass:, interval: 10, inhibitions: [], reaction: nil, **opts)
|
|
16
16
|
@app = app
|
|
17
17
|
@killer = killer
|
|
18
|
+
@interval = interval
|
|
18
19
|
|
|
19
|
-
@inhibitions =
|
|
20
|
-
inhibitions.dup.freeze
|
|
21
|
-
else
|
|
22
|
-
[].freeze
|
|
23
|
-
end
|
|
20
|
+
@inhibitions = inhibitions.dup.freeze
|
|
24
21
|
|
|
25
22
|
@reaction = reaction || method(:default_kill)
|
|
26
23
|
|
|
27
24
|
@limiter = klass.new(**opts)
|
|
25
|
+
@last_reacted_at = 0.0
|
|
26
|
+
@inhibited = 0
|
|
27
|
+
@delayed_reaction = nil
|
|
28
28
|
end
|
|
29
29
|
|
|
30
30
|
def call(env)
|
|
31
|
-
inhibited = false
|
|
32
31
|
if (path_info = env['PATH_INFO']) && inhibitions.any?{|i| path_info[i] }
|
|
33
|
-
|
|
34
|
-
|
|
32
|
+
call_with_inhibition(env, path_info) do
|
|
33
|
+
# реакция будет вызвана после реального окончания обработки
|
|
34
|
+
react if limiter.check
|
|
35
|
+
end
|
|
36
|
+
else
|
|
37
|
+
@app.call(env).tap do
|
|
38
|
+
# реакция будет вызвана сейчас (как обычно)
|
|
39
|
+
react if limiter.check
|
|
40
|
+
end
|
|
35
41
|
end
|
|
42
|
+
end
|
|
36
43
|
|
|
37
|
-
|
|
38
|
-
|
|
44
|
+
def call_with_inhibition(env, path_info, &after)
|
|
45
|
+
inihibit(path_info)
|
|
39
46
|
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
47
|
+
rack_response = nil
|
|
48
|
+
begin
|
|
49
|
+
rack_response = @app.call(env)
|
|
50
|
+
rescue Exception
|
|
51
|
+
# в случае ошибоки во время @app.call возращаем как было
|
|
52
|
+
release
|
|
53
|
+
raise
|
|
54
|
+
end
|
|
44
55
|
|
|
45
|
-
def with_inhibition(tuple)
|
|
46
56
|
# Почему именно each описано в спецификации в разделе The Response
|
|
47
57
|
# https://github.com/rack/rack/blob/main/SPEC.rdoc
|
|
48
|
-
if
|
|
49
|
-
|
|
50
|
-
tuple[2] = Enumerator.new do |y|
|
|
51
|
-
old.each do |chunk|
|
|
52
|
-
y << chunk
|
|
53
|
-
end
|
|
54
|
-
old.close if old.respond_to?(:close)
|
|
55
|
-
release
|
|
56
|
-
end
|
|
58
|
+
if rack_response[2].respond_to?(:each)
|
|
59
|
+
rack_response[2] = wrap_rack_response_body(rack_response[2], &after)
|
|
57
60
|
else
|
|
58
|
-
release
|
|
61
|
+
release # освобождаем сразу
|
|
62
|
+
after.call
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
rack_response
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def wrap_rack_response_body(rack_response_body, &after)
|
|
69
|
+
Enumerator.new do |y|
|
|
70
|
+
rack_response_body.each {|chunk| y << chunk }
|
|
71
|
+
# Почему именно close описано в спецификации в разделе The Response
|
|
72
|
+
# https://github.com/rack/rack/blob/main/SPEC.rdoc
|
|
73
|
+
rack_response_body.close if rack_response_body.respond_to?(:close)
|
|
74
|
+
|
|
75
|
+
release # освобождаем после отправки всего тела
|
|
76
|
+
after.call
|
|
77
|
+
end
|
|
78
|
+
end
|
|
79
|
+
|
|
80
|
+
def react
|
|
81
|
+
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
82
|
+
return if now - @last_reacted_at < @interval
|
|
83
|
+
|
|
84
|
+
@last_reacted_at = now
|
|
85
|
+
if @inhibited > 0
|
|
86
|
+
@delayed_reaction = -> { reaction.call(limiter, killer) }
|
|
87
|
+
else
|
|
88
|
+
reaction.call(limiter, killer)
|
|
59
89
|
end
|
|
60
|
-
tuple
|
|
61
90
|
end
|
|
62
91
|
|
|
63
92
|
def default_kill(l, k)
|
|
64
93
|
k.kill(l.started_at)
|
|
65
94
|
end
|
|
66
95
|
|
|
67
|
-
def inihibit(
|
|
68
|
-
|
|
69
|
-
rescue StandardError => e
|
|
70
|
-
logger.error "#{self.class}::inhibit error: #{e.inspect}"
|
|
96
|
+
def inihibit(_path_info)
|
|
97
|
+
@inhibited += 1
|
|
71
98
|
end
|
|
72
99
|
|
|
73
100
|
def release
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
101
|
+
return unless ((@inhibited -= 1) == 0) && @delayed_reaction
|
|
102
|
+
|
|
103
|
+
@delayed_reaction.tap do |cb|
|
|
104
|
+
@delayed_reaction = nil
|
|
105
|
+
cb.call
|
|
106
|
+
end
|
|
77
107
|
end
|
|
78
108
|
|
|
79
109
|
def logger
|
|
@@ -42,6 +42,7 @@ module WorkerKiller
|
|
|
42
42
|
cb = if dsl.respond_to?(:before_worker_boot)
|
|
43
43
|
:before_worker_boot
|
|
44
44
|
else
|
|
45
|
+
# DEPRECATED
|
|
45
46
|
:on_worker_boot
|
|
46
47
|
end
|
|
47
48
|
|
|
@@ -64,6 +65,7 @@ module WorkerKiller
|
|
|
64
65
|
cb = if launcher.events.respond_to?(:after_booted)
|
|
65
66
|
:after_booted
|
|
66
67
|
else
|
|
68
|
+
# DEPRECATED
|
|
67
69
|
:on_booted
|
|
68
70
|
end
|
|
69
71
|
|
|
@@ -22,7 +22,7 @@ module WorkerKiller
|
|
|
22
22
|
|
|
23
23
|
end
|
|
24
24
|
|
|
25
|
-
attr_accessor :killer, :puma_server, :
|
|
25
|
+
attr_accessor :killer, :puma_server, :kill_queue
|
|
26
26
|
|
|
27
27
|
def initialize
|
|
28
28
|
@killer = ::WorkerKiller::Killer::Puma.new(worker_num: nil, puma_plugin: self)
|
|
@@ -31,9 +31,9 @@ module WorkerKiller
|
|
|
31
31
|
|
|
32
32
|
@puma_server = nil
|
|
33
33
|
|
|
34
|
-
@force_restart = false
|
|
35
|
-
@inhibited ||= Hash.new {|h, k| h[k] = 0 }
|
|
34
|
+
@force_restart = %w[t 1].include?(ENV.fetch('WORKER_KILLER_PUMA_AGGRESSIVE', 'false').to_s.downcase[0].to_s)
|
|
36
35
|
@kill_queue ||= Set.new
|
|
36
|
+
@last_restarted_at = 0.0
|
|
37
37
|
end
|
|
38
38
|
|
|
39
39
|
# Этот метод зовётся при ИНИЦИАЛИЗАЦИИ плагина внути master-процесса, в самомо начале
|
|
@@ -46,9 +46,12 @@ module WorkerKiller
|
|
|
46
46
|
cb = if dsl.respond_to?(:before_worker_boot)
|
|
47
47
|
:before_worker_boot
|
|
48
48
|
else
|
|
49
|
+
# DEPRECATED
|
|
49
50
|
:on_worker_boot
|
|
50
51
|
end
|
|
51
52
|
|
|
53
|
+
puts "SEND:#{cb}"
|
|
54
|
+
|
|
52
55
|
dsl.send(cb) do |num|
|
|
53
56
|
@killer.worker_num = num
|
|
54
57
|
@worker_num = num
|
|
@@ -80,37 +83,22 @@ module WorkerKiller
|
|
|
80
83
|
def request_restart_server(worker_num)
|
|
81
84
|
return if @worker_num != worker_num
|
|
82
85
|
|
|
83
|
-
log("
|
|
86
|
+
log("Enqueue worker #{worker_num} for restarting...")
|
|
84
87
|
kill_queue << worker_num
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
# Этот метод зовётся из Middleware внтури воркера
|
|
88
|
-
def inhibit_restart(worker_num)
|
|
89
|
-
return if @worker_num != worker_num
|
|
90
|
-
|
|
91
|
-
cnt = inhibited[worker_num] += 1 # just increase inhibit counter
|
|
92
|
-
debug("Worker inhibition increased: #{cnt}")
|
|
93
|
-
end
|
|
94
|
-
|
|
95
|
-
# Этот метод зовётся из Middleware внтури воркера
|
|
96
|
-
def release_restart(worker_num)
|
|
97
|
-
return if @worker_num != worker_num
|
|
98
|
-
|
|
99
|
-
cnt = inhibited[worker_num] -= 1 # just decrease inhibit counter
|
|
100
|
-
debug("Worker inhibition decreased: #{cnt}")
|
|
101
|
-
return unless cnt <= 0
|
|
102
|
-
|
|
103
|
-
inhibited.delete(worker_num)
|
|
104
|
-
debug('Worker released')
|
|
105
|
-
do_kill('RELEASE') if @force_restart
|
|
88
|
+
do_kill('FORCE') if @force_restart
|
|
106
89
|
end
|
|
107
90
|
|
|
108
91
|
def do_kill(name)
|
|
109
92
|
return if kill_queue.empty?
|
|
110
93
|
|
|
94
|
+
now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
95
|
+
return if now - @last_restarted_at < 10
|
|
96
|
+
|
|
97
|
+
@last_restarted_at = now
|
|
111
98
|
log "Killing workers by #{name}: #{kill_queue}"
|
|
112
|
-
|
|
99
|
+
kill_queue.each do |worker_num|
|
|
113
100
|
kill_queue.delete(worker_num)
|
|
101
|
+
Thread.current.puma_server.options[:force_shutdown_after] = nil
|
|
114
102
|
Thread.current.puma_server.begin_restart
|
|
115
103
|
end
|
|
116
104
|
end
|
|
@@ -1,35 +1,16 @@
|
|
|
1
1
|
RSpec.describe WorkerKiller::Killer::DelayedJob do
|
|
2
|
-
let(:config) do
|
|
3
|
-
WorkerKiller::Configuration.new.tap do |c|
|
|
4
|
-
c.quit_attempts = 2
|
|
5
|
-
c.term_attempts = 2
|
|
6
|
-
end
|
|
7
|
-
end
|
|
8
|
-
|
|
9
2
|
let(:dj){ double }
|
|
10
|
-
subject(:killer){ described_class.new
|
|
3
|
+
subject(:killer){ described_class.new }
|
|
11
4
|
|
|
12
5
|
describe '#kill' do
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
example.run
|
|
18
|
-
ensure
|
|
19
|
-
WorkerKiller.configuration = prev
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
it 'expect right signal order' do
|
|
23
|
-
expect(dj).to receive(:stop).exactly(4)
|
|
24
|
-
expect(Process).to receive(:kill).with(:TERM, anything).exactly(1).times
|
|
25
|
-
expect(Process).to receive(:kill).with(:KILL, anything).exactly(5).times
|
|
6
|
+
it 'expect right signal order' do
|
|
7
|
+
expect(dj).to receive(:stop).exactly(1)
|
|
8
|
+
expect(Process).to receive(:kill).with(:TERM, anything).exactly(1).times
|
|
9
|
+
expect(Process).to receive(:kill).with(:KILL, anything).exactly(6).times
|
|
26
10
|
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
5.times { killer.kill(Time.now, dj: dj) } # 5 KILL
|
|
30
|
-
end
|
|
11
|
+
3.times { killer.kill(Time.now, dj: dj) } # 1 QUIT 1 TERM 1 KILL
|
|
12
|
+
5.times { killer.kill(Time.now, dj: dj) } # 5 KILL
|
|
31
13
|
end
|
|
32
14
|
end
|
|
33
|
-
|
|
34
15
|
end
|
|
35
16
|
|
|
@@ -1,42 +1,25 @@
|
|
|
1
1
|
RSpec.describe WorkerKiller::Killer::Passenger do
|
|
2
|
-
let(:
|
|
3
|
-
WorkerKiller::Configuration.new.tap do |c|
|
|
4
|
-
c.quit_attempts = 2
|
|
5
|
-
c.term_attempts = 2
|
|
6
|
-
end
|
|
7
|
-
end
|
|
8
|
-
|
|
9
|
-
let(:killer){ described_class.new() }
|
|
2
|
+
let(:killer){ described_class.new }
|
|
10
3
|
|
|
11
4
|
describe '#kill' do
|
|
12
5
|
before do
|
|
13
6
|
allow(described_class).to receive(:check_passenger_config).and_return('custompath')
|
|
14
7
|
end
|
|
15
8
|
|
|
16
|
-
around do |example|
|
|
17
|
-
prev = WorkerKiller.configuration
|
|
18
|
-
WorkerKiller.configuration = config
|
|
19
|
-
example.run
|
|
20
|
-
ensure
|
|
21
|
-
WorkerKiller.configuration = prev
|
|
22
|
-
end
|
|
23
|
-
|
|
24
9
|
it 'expect right signal order' do
|
|
25
10
|
expect(Kernel).to receive(:system).with("custompath detach-process #{Process.pid}").and_return(true)
|
|
26
|
-
expect(Process).to receive(:kill).with(:KILL, anything).exactly(
|
|
27
|
-
|
|
28
|
-
thread = killer.kill(Time.now)
|
|
29
|
-
thread.join
|
|
11
|
+
expect(Process).to receive(:kill).with(:KILL, anything).exactly(6).times
|
|
30
12
|
|
|
31
|
-
|
|
32
|
-
2.times { killer.kill(Time.now) } # 1 TERM
|
|
13
|
+
3.times { killer.kill(Time.now) } # 1 QUIT 1 TERM 1 KILL
|
|
33
14
|
5.times { killer.kill(Time.now) } # 5 KILL
|
|
34
15
|
end
|
|
35
16
|
end
|
|
36
17
|
|
|
37
18
|
describe '#check_passenger_config!' do
|
|
38
19
|
it do
|
|
39
|
-
expect
|
|
20
|
+
expect do
|
|
21
|
+
described_class.check_passenger_config!('nonenone')
|
|
22
|
+
end.to raise_error(/Can't find passenger/)
|
|
40
23
|
end
|
|
41
24
|
end
|
|
42
25
|
end
|
data/spec/killer/puma_spec.rb
CHANGED
|
@@ -10,21 +10,5 @@ RSpec.describe WorkerKiller::Killer::Puma do
|
|
|
10
10
|
end
|
|
11
11
|
end
|
|
12
12
|
|
|
13
|
-
describe '#do_inhibit' do
|
|
14
|
-
it do
|
|
15
|
-
expect(puma_plugin).to receive(:inhibit_restart)
|
|
16
|
-
|
|
17
|
-
killer.do_inhibit('something')
|
|
18
|
-
end
|
|
19
|
-
end
|
|
20
|
-
|
|
21
|
-
describe '#do_release' do
|
|
22
|
-
it do
|
|
23
|
-
expect(puma_plugin).to receive(:release_restart)
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
killer.do_release
|
|
27
|
-
end
|
|
28
|
-
end
|
|
29
13
|
end
|
|
30
14
|
|
data/spec/killer/signal_spec.rb
CHANGED
|
@@ -1,32 +1,14 @@
|
|
|
1
1
|
RSpec.describe WorkerKiller::Killer::Signal do
|
|
2
|
-
let(:
|
|
3
|
-
WorkerKiller::Configuration.new.tap do |c|
|
|
4
|
-
c.quit_attempts = 2
|
|
5
|
-
c.term_attempts = 2
|
|
6
|
-
end
|
|
7
|
-
end
|
|
8
|
-
|
|
9
|
-
let(:killer){ described_class.new() }
|
|
2
|
+
let(:killer){ described_class.new }
|
|
10
3
|
|
|
11
4
|
describe '#kill' do
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
example.run
|
|
17
|
-
ensure
|
|
18
|
-
WorkerKiller.configuration = prev
|
|
19
|
-
end
|
|
20
|
-
|
|
21
|
-
it 'expect right signal order' do
|
|
22
|
-
expect(Process).to receive(:kill).with(:QUIT, anything).exactly(1).times
|
|
23
|
-
expect(Process).to receive(:kill).with(:TERM, anything).exactly(1).times
|
|
24
|
-
expect(Process).to receive(:kill).with(:KILL, anything).exactly(1).times
|
|
5
|
+
it 'expect right signal order' do
|
|
6
|
+
expect(Process).to receive(:kill).with(:QUIT, anything).exactly(1).times
|
|
7
|
+
expect(Process).to receive(:kill).with(:TERM, anything).exactly(1).times
|
|
8
|
+
expect(Process).to receive(:kill).with(:KILL, anything).exactly(1).times
|
|
25
9
|
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
5.times { killer.kill(Time.now) } # 1 KILL
|
|
29
|
-
end
|
|
10
|
+
3.times { killer.kill(Time.now) } # 1 QUIT 1 TERM 1 KILL
|
|
11
|
+
5.times { killer.kill(Time.now) } # nothing
|
|
30
12
|
end
|
|
31
13
|
end
|
|
32
14
|
end
|
data/spec/killer_spec.rb
CHANGED
|
@@ -1,40 +1,24 @@
|
|
|
1
1
|
RSpec.describe WorkerKiller::Killer::Base do
|
|
2
|
-
let(:
|
|
3
|
-
WorkerKiller::Configuration.new.tap do |c|
|
|
4
|
-
c.quit_attempts = 2
|
|
5
|
-
c.term_attempts = 2
|
|
6
|
-
end
|
|
7
|
-
end
|
|
8
|
-
|
|
9
|
-
let(:killer){ described_class.new() }
|
|
2
|
+
let(:killer){ described_class.new }
|
|
10
3
|
|
|
11
4
|
describe '#kill' do
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
|
|
5
|
+
it 'expect right signal order' do
|
|
6
|
+
if RUBY_VERSION >= '2.7.0'
|
|
7
|
+
expect(killer).to receive(:do_kill).with(:QUIT, anything, anything).exactly(1).times
|
|
8
|
+
expect(killer).to receive(:do_kill).with(:TERM, anything, anything).exactly(1).times
|
|
9
|
+
expect(killer).to receive(:do_kill).with(:KILL, anything, anything).exactly(6).times
|
|
10
|
+
else
|
|
11
|
+
expect(killer).to receive(:do_kill).with(:QUIT, anything, anything,
|
|
12
|
+
anything).exactly(1).times
|
|
13
|
+
expect(killer).to receive(:do_kill).with(:TERM, anything, anything,
|
|
14
|
+
anything).exactly(1).times
|
|
15
|
+
expect(killer).to receive(:do_kill).with(:KILL, anything, anything,
|
|
16
|
+
anything).exactly(6).times
|
|
19
17
|
end
|
|
20
18
|
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
expect(killer).to receive(:do_kill).with(:QUIT, anything, anything).exactly(2).times
|
|
24
|
-
expect(killer).to receive(:do_kill).with(:TERM, anything, anything).exactly(2).times
|
|
25
|
-
expect(killer).to receive(:do_kill).with(:KILL, anything, anything).exactly(5).times
|
|
26
|
-
else
|
|
27
|
-
expect(killer).to receive(:do_kill).with(:QUIT, anything, anything, anything).exactly(2).times
|
|
28
|
-
expect(killer).to receive(:do_kill).with(:TERM, anything, anything, anything).exactly(2).times
|
|
29
|
-
expect(killer).to receive(:do_kill).with(:KILL, anything, anything, anything).exactly(5).times
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
2.times { killer.kill(Time.now) } # 2 QUIT
|
|
33
|
-
2.times { killer.kill(Time.now) } # 2 TERM
|
|
34
|
-
5.times { killer.kill(Time.now) } # other - KILL
|
|
35
|
-
end
|
|
19
|
+
3.times { killer.kill(Time.now) } # 1 QUIT, 1 TERM, 1 KILL
|
|
20
|
+
5.times { killer.kill(Time.now) } # 5 KILL
|
|
36
21
|
end
|
|
37
|
-
|
|
38
22
|
end
|
|
39
23
|
end
|
|
40
24
|
|
data/spec/middleware_spec.rb
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
require 'securerandom'
|
|
2
2
|
|
|
3
3
|
RSpec.describe WorkerKiller::Middleware do
|
|
4
|
-
let(:
|
|
4
|
+
let(:rack_body) { double(each: %w[1 2 3], close: true) }
|
|
5
|
+
let(:app){ double(call: [nil, nil, rack_body]) }
|
|
5
6
|
let(:killer) { double(WorkerKiller::Killer::Base, do_inhibit: true) }
|
|
6
7
|
let(:reaction){ double }
|
|
7
8
|
let(:anykey){ SecureRandom.hex(8) }
|
|
@@ -9,14 +10,15 @@ RSpec.describe WorkerKiller::Middleware do
|
|
|
9
10
|
let(:env) { { 'PATH_INFO' => '/something' } }
|
|
10
11
|
|
|
11
12
|
describe 'Custom class' do
|
|
12
|
-
let(:
|
|
13
|
+
let(:limiter_klass){ double }
|
|
13
14
|
let(:options) do
|
|
14
|
-
{ killer: killer, klass:
|
|
15
|
+
{ killer: killer, klass: limiter_klass, reaction: reaction, anykey: anykey,
|
|
16
|
+
inhibitions: inhibitions }
|
|
15
17
|
end
|
|
16
18
|
subject{ described_class.new(app, **options) }
|
|
17
19
|
|
|
18
20
|
it 'is expected to be initialized' do
|
|
19
|
-
expect(
|
|
21
|
+
expect(limiter_klass).to receive(:new).with(anykey: anykey).and_return(99)
|
|
20
22
|
expect(subject.limiter).to eq(99)
|
|
21
23
|
expect(subject.killer).to eq(killer)
|
|
22
24
|
end
|
|
@@ -31,14 +33,13 @@ RSpec.describe WorkerKiller::Middleware do
|
|
|
31
33
|
expect(subject.limiter).to be_an(WorkerKiller::CountLimiter)
|
|
32
34
|
expect(subject.limiter.min).to eq(3)
|
|
33
35
|
expect(subject.limiter.max).to eq(3)
|
|
34
|
-
expect(killer).to receive(:kill).with(Time)
|
|
36
|
+
expect(killer).to receive(:kill).with(Time)
|
|
35
37
|
|
|
36
38
|
4.times do
|
|
37
39
|
subject.call(env)
|
|
38
40
|
end
|
|
39
41
|
end
|
|
40
42
|
|
|
41
|
-
|
|
42
43
|
describe 'inhibitions' do
|
|
43
44
|
let(:env) { { 'PATH_INFO' => '/attachments' } }
|
|
44
45
|
|
|
@@ -47,11 +48,14 @@ RSpec.describe WorkerKiller::Middleware do
|
|
|
47
48
|
expect(subject.limiter).to be_an(WorkerKiller::CountLimiter)
|
|
48
49
|
expect(subject.limiter.min).to eq(3)
|
|
49
50
|
expect(subject.limiter.max).to eq(3)
|
|
50
|
-
expect(killer).to receive(:kill).with(Time)
|
|
51
|
-
expect(
|
|
51
|
+
expect(killer).to receive(:kill).with(Time)
|
|
52
|
+
expect(subject).to receive(:release).exactly(4).times.and_call_original
|
|
52
53
|
|
|
53
54
|
4.times do
|
|
54
|
-
|
|
55
|
+
expect(app).to receive(:call).and_return([nil, nil, rack_body])
|
|
56
|
+
rack_response = subject.call(env)
|
|
57
|
+
subject.react
|
|
58
|
+
rack_response[2].each.to_a
|
|
55
59
|
end
|
|
56
60
|
end
|
|
57
61
|
end
|
|
@@ -63,7 +67,7 @@ RSpec.describe WorkerKiller::Middleware do
|
|
|
63
67
|
expect(subject.limiter).to be_an(WorkerKiller::CountLimiter)
|
|
64
68
|
expect(subject.limiter.min).to eq(3)
|
|
65
69
|
expect(subject.limiter.max).to eq(3)
|
|
66
|
-
expect(reaction).to receive(:call).with(subject.limiter, killer)
|
|
70
|
+
expect(reaction).to receive(:call).with(subject.limiter, killer)
|
|
67
71
|
|
|
68
72
|
4.times do
|
|
69
73
|
subject.call(env)
|
data/spec/puma_plugin_ng_spec.rb
CHANGED
|
@@ -8,7 +8,6 @@ RSpec.describe WorkerKiller::PumaPluginNg do
|
|
|
8
8
|
|
|
9
9
|
it {
|
|
10
10
|
is_expected.to have_attributes(killer: ::WorkerKiller::Killer::Puma,
|
|
11
|
-
inhibited: Hash,
|
|
12
11
|
kill_queue: Set)
|
|
13
12
|
}
|
|
14
13
|
|
|
@@ -32,29 +31,13 @@ RSpec.describe WorkerKiller::PumaPluginNg do
|
|
|
32
31
|
end
|
|
33
32
|
end
|
|
34
33
|
|
|
35
|
-
context '#inhibit_restart' do
|
|
36
|
-
it do
|
|
37
|
-
plugin.instance_variable_set('@worker_num', 99)
|
|
38
|
-
expect { plugin.inhibit_restart(99) }.to change { plugin.inhibited }.from({}).to({ 99 => 1 })
|
|
39
|
-
end
|
|
40
|
-
end
|
|
41
|
-
|
|
42
|
-
context '#release_restart' do
|
|
43
|
-
it do
|
|
44
|
-
plugin.instance_variable_set('@worker_num', 99)
|
|
45
|
-
plugin.inhibit_restart(99)
|
|
46
|
-
|
|
47
|
-
expect { plugin.release_restart(99) }.to change { plugin.inhibited }.from({ 99 => 1 }).to({})
|
|
48
|
-
end
|
|
49
|
-
end
|
|
50
|
-
|
|
51
34
|
context '#do_kill' do
|
|
52
35
|
it do
|
|
53
36
|
Thread.attr_accessor(:puma_server)
|
|
54
37
|
|
|
55
38
|
plugin.instance_variable_set('@worker_num', 99)
|
|
56
39
|
plugin.request_restart_server(99)
|
|
57
|
-
puma_server = double()
|
|
40
|
+
puma_server = double(options: {})
|
|
58
41
|
Thread.current.puma_server = puma_server
|
|
59
42
|
expect(puma_server).to receive(:begin_restart)
|
|
60
43
|
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: worker_killer
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.2.
|
|
4
|
+
version: 1.2.1.338700
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Samoilenko Yuri
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2025-11-
|
|
11
|
+
date: 2025-11-14 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: get_process_mem
|