worker_killer 0.1.1.30345 → 0.1.1.39838
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/worker_killer.rb +40 -39
- data/lib/worker_killer/configuration.rb +3 -35
- data/lib/worker_killer/count_limiter.rb +1 -4
- data/lib/worker_killer/delayed_job_plugin.rb +31 -0
- data/lib/worker_killer/killer.rb +38 -0
- data/lib/worker_killer/killer/delayed_job.rb +27 -0
- data/lib/worker_killer/killer/passenger.rb +55 -0
- data/lib/worker_killer/killer/signal.rb +16 -0
- data/lib/worker_killer/memory_limiter.rb +1 -4
- data/lib/worker_killer/middleware.rb +7 -8
- data/spec/count_limiter_spec.rb +6 -14
- data/spec/killer/delayed_job_spec.rb +34 -0
- data/spec/killer/passenger_spec.rb +43 -0
- data/spec/killer/signal_spec.rb +33 -0
- data/spec/killer_spec.rb +34 -0
- data/spec/memory_limiter_spec.rb +6 -15
- data/spec/middleware_spec.rb +47 -12
- data/spec/spec_helper.rb +1 -0
- data/spec/support/logger.rb +12 -0
- data/spec/worker_killer_spec.rb +0 -73
- metadata +17 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 92604c2f0b3b1fbd27a647f8c72556abbd2c6a38d3e57a120697118db1921b44
|
|
4
|
+
data.tar.gz: e22fb98b3d5dd0781e79eaba94213e7cca03c728f0f64ab0fe391b4f536de850
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: acabb632791e185856f983c44b3f7ae151e5798508e846c669b8699ec6bf06bd435c6a4a9fe551e94033e5a577ab252a8a3aaaffb09733ee52a27422faf0de0a
|
|
7
|
+
data.tar.gz: 253ae056f4b4d107458a85a897c3ad16a6ec532282bf7e9781fed06cfde943f177b23c6c3f96c3c513fe6f4a01ed92604c84eaca31bc64dca05ccfe2ba9c902e
|
data/lib/worker_killer.rb
CHANGED
|
@@ -3,6 +3,7 @@ require 'worker_killer/configuration'
|
|
|
3
3
|
require 'worker_killer/count_limiter'
|
|
4
4
|
require 'worker_killer/memory_limiter'
|
|
5
5
|
require 'worker_killer/middleware'
|
|
6
|
+
require 'worker_killer/killer'
|
|
6
7
|
|
|
7
8
|
module WorkerKiller
|
|
8
9
|
|
|
@@ -27,45 +28,45 @@ module WorkerKiller
|
|
|
27
28
|
# the process isn't killed after `configuration.quit_attempts` QUIT signals,
|
|
28
29
|
# send TERM signals until `configuration.kill_attempts`. Finally, send a KILL
|
|
29
30
|
# signal. A single signal is sent per request.
|
|
30
|
-
def self.kill_self(logger, start_time)
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
end
|
|
51
|
-
|
|
52
|
-
def self.kill_by_signal(logger, alive_sec, signal, pid)
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
end
|
|
56
|
-
|
|
57
|
-
def self.kill_by_passenger(logger, alive_sec, passenger, pid)
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
end
|
|
31
|
+
# def self.kill_self(logger, start_time)
|
|
32
|
+
# alive_sec = (Time.now - start_time).round
|
|
33
|
+
|
|
34
|
+
# @kill_attempts ||= 0
|
|
35
|
+
# @kill_attempts += 1
|
|
36
|
+
|
|
37
|
+
# if configuration.use_quit
|
|
38
|
+
# sig = :QUIT
|
|
39
|
+
# sig = :TERM if @kill_attempts > configuration.quit_attempts
|
|
40
|
+
# sig = :KILL if @kill_attempts > (configuration.quit_attempts + configuration.term_attempts)
|
|
41
|
+
# else
|
|
42
|
+
# sig = :TERM
|
|
43
|
+
# sig = :KILL if @kill_attempts > configuration.term_attempts
|
|
44
|
+
# end
|
|
45
|
+
|
|
46
|
+
# if sig == :QUIT && configuration.passenger?
|
|
47
|
+
# kill_by_passenger(logger, alive_sec, configuration.passenger_config, Process.pid)
|
|
48
|
+
# else
|
|
49
|
+
# kill_by_signal(logger, alive_sec, sig, Process.pid)
|
|
50
|
+
# end
|
|
51
|
+
# end
|
|
52
|
+
|
|
53
|
+
# def self.kill_by_signal(logger, alive_sec, signal, pid)
|
|
54
|
+
# logger.warn "#{self} send SIG#{signal} (pid: #{pid}) alive: #{alive_sec} sec (trial #{@kill_attempts})"
|
|
55
|
+
# Process.kill signal, pid
|
|
56
|
+
# end
|
|
57
|
+
|
|
58
|
+
# def self.kill_by_passenger(logger, alive_sec, passenger, pid)
|
|
59
|
+
# return if @already_detached
|
|
60
|
+
# @already_detached = true
|
|
61
|
+
|
|
62
|
+
# cmd = "#{passenger} detach-process #{pid}"
|
|
63
|
+
# logger.warn "#{self} run #{cmd.inspect} (pid: #{pid}) alive: #{alive_sec} sec (trial #{@kill_attempts})"
|
|
64
|
+
# Thread.new(cmd) do |command|
|
|
65
|
+
# unless Kernel.system(command)
|
|
66
|
+
# logger.warn "#{self} run #{cmd.inspect} failed: #{$?.inspect}"
|
|
67
|
+
# end
|
|
68
|
+
# end
|
|
69
|
+
# end
|
|
69
70
|
|
|
70
71
|
end
|
|
71
72
|
|
|
@@ -4,47 +4,15 @@ module WorkerKiller
|
|
|
4
4
|
# Methods for configuring WorkerKiller
|
|
5
5
|
class Configuration
|
|
6
6
|
|
|
7
|
-
attr_accessor :logger, :quit_attempts, :
|
|
7
|
+
attr_accessor :logger, :quit_attempts, :term_attempts
|
|
8
8
|
|
|
9
9
|
# Override defaults for configuration
|
|
10
|
-
def initialize(quit_attempts:
|
|
10
|
+
def initialize(quit_attempts: 10, term_attempts: 50)
|
|
11
11
|
@quit_attempts = quit_attempts
|
|
12
|
-
@
|
|
13
|
-
@use_quit = use_quit
|
|
14
|
-
@passenger_config = check_passenger(passenger_config)
|
|
12
|
+
@term_attempts = term_attempts
|
|
15
13
|
@logger = Logger.new(STDOUT, level: Logger::INFO, progname: 'WorkerKiller')
|
|
16
14
|
end
|
|
17
15
|
|
|
18
|
-
def passenger?
|
|
19
|
-
!@passenger_config.nil?
|
|
20
|
-
end
|
|
21
|
-
|
|
22
|
-
def check_passenger provided_path
|
|
23
|
-
if provided_path.nil? || provided_path.empty?
|
|
24
|
-
return check_passenger_config(`which passenger-config`)
|
|
25
|
-
else
|
|
26
|
-
return check_passenger_config!(provided_path)
|
|
27
|
-
end
|
|
28
|
-
end
|
|
29
|
-
|
|
30
|
-
def check_passenger_config path
|
|
31
|
-
path.strip!
|
|
32
|
-
help = `#{path} detach-process --help 2> /dev/null`
|
|
33
|
-
if help['Remove an application process from the Phusion Passenger process pool']
|
|
34
|
-
return path
|
|
35
|
-
end
|
|
36
|
-
rescue StandardError => e
|
|
37
|
-
return nil
|
|
38
|
-
end
|
|
39
|
-
|
|
40
|
-
def check_passenger_config! path
|
|
41
|
-
if path = check_passenger_config(path)
|
|
42
|
-
return path
|
|
43
|
-
else
|
|
44
|
-
raise "Can't find passenger config at #{path.inspect}"
|
|
45
|
-
end
|
|
46
|
-
end
|
|
47
|
-
|
|
48
16
|
end
|
|
49
17
|
end
|
|
50
18
|
|
|
@@ -2,15 +2,13 @@ module WorkerKiller
|
|
|
2
2
|
class CountLimiter
|
|
3
3
|
|
|
4
4
|
attr_reader :min, :max, :left, :limit, :started_at
|
|
5
|
-
attr_accessor :reaction
|
|
6
5
|
|
|
7
|
-
def initialize(min: 3072, max: 4096, verbose: false
|
|
6
|
+
def initialize(min: 3072, max: 4096, verbose: false)
|
|
8
7
|
@min = min
|
|
9
8
|
@max = max
|
|
10
9
|
@limit = @min + WorkerKiller.randomize(@max - @min + 1)
|
|
11
10
|
@left = @limit
|
|
12
11
|
@verbose = verbose
|
|
13
|
-
@reaction = block
|
|
14
12
|
end
|
|
15
13
|
|
|
16
14
|
def check
|
|
@@ -25,7 +23,6 @@ module WorkerKiller
|
|
|
25
23
|
return false if (@left -= 1) > 0
|
|
26
24
|
|
|
27
25
|
logger.warn "#{self}: worker (pid: #{Process.pid}) exceeds max number of requests (limit: #{@limit})"
|
|
28
|
-
@reaction.call(self)
|
|
29
26
|
|
|
30
27
|
true
|
|
31
28
|
end
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
require 'worker_killer/memory_limiter'
|
|
2
|
+
require 'worker_killer/count_limiter'
|
|
3
|
+
|
|
4
|
+
module WorkerKiller
|
|
5
|
+
class DelayedJobPlugin
|
|
6
|
+
|
|
7
|
+
attr_reader :limiter, :killer, :reaction
|
|
8
|
+
|
|
9
|
+
def initialize(klass:, killer:, reaction: nil, **opts)
|
|
10
|
+
@killer = killer
|
|
11
|
+
|
|
12
|
+
@reaction = reaction || proc do |l, k, dj|
|
|
13
|
+
k.kill(l.started_at, dj: dj)
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
@limiter = klass.new(opts)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def new(*_args)
|
|
20
|
+
configure_lifecycle(Delayed::Worker.lifecycle)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def configure_lifecycle(lifecycle)
|
|
24
|
+
lifecycle.after(:perform) do |worker, *_args|
|
|
25
|
+
reaction.call(limiter, killer, worker) if limiter.check
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
module WorkerKiller
|
|
2
|
+
module Killer
|
|
3
|
+
class Base
|
|
4
|
+
|
|
5
|
+
attr_accessor :config, :kill_attempts, :logger
|
|
6
|
+
|
|
7
|
+
def initialize(logger: WorkerKiller.configuration.logger, **_kwargs)
|
|
8
|
+
@logger = logger
|
|
9
|
+
@config = WorkerKiller.configuration
|
|
10
|
+
@kill_attempts = 0
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
def kill(start_time, **params)
|
|
14
|
+
alive_sec = (Time.now - start_time).round
|
|
15
|
+
|
|
16
|
+
@kill_attempts += 1
|
|
17
|
+
|
|
18
|
+
sig = :QUIT
|
|
19
|
+
sig = :TERM if kill_attempts > config.quit_attempts
|
|
20
|
+
sig = :KILL if kill_attempts > (config.quit_attempts + config.term_attempts)
|
|
21
|
+
|
|
22
|
+
do_kill(sig, Process.pid, alive_sec, **params)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
# :nocov:
|
|
26
|
+
def do_kill(*_args)
|
|
27
|
+
raise 'Not Implemented'
|
|
28
|
+
end
|
|
29
|
+
# :nocov:
|
|
30
|
+
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
require_relative 'killer/signal'
|
|
36
|
+
require_relative 'killer/passenger'
|
|
37
|
+
require_relative 'killer/delayed_job'
|
|
38
|
+
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
module WorkerKiller
|
|
2
|
+
module Killer
|
|
3
|
+
class DelayedJob < ::WorkerKiller::Killer::Base
|
|
4
|
+
|
|
5
|
+
def do_kill(sig, pid, alive_sec, dj:, **_params)
|
|
6
|
+
if sig == :KILL
|
|
7
|
+
logger.error "#{self} force to #{sig} self (pid: #{pid}) alive: #{alive_sec} sec (trial #{kill_attempts})"
|
|
8
|
+
Process.kill sig, pid
|
|
9
|
+
return
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
dj.stop
|
|
13
|
+
|
|
14
|
+
return if sig != :TERM
|
|
15
|
+
|
|
16
|
+
if @termination
|
|
17
|
+
logger.warn "#{self} force to #{sig} self (pid: #{pid}) alive: #{alive_sec} sec (trial #{kill_attempts})"
|
|
18
|
+
Process.kill sig, pid
|
|
19
|
+
else
|
|
20
|
+
@termination = true
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
end
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
module WorkerKiller
|
|
2
|
+
module Killer
|
|
3
|
+
class Passenger < ::WorkerKiller::Killer::Base
|
|
4
|
+
|
|
5
|
+
attr_reader :passenger_config
|
|
6
|
+
|
|
7
|
+
def initialize path: nil, **kwrags
|
|
8
|
+
super
|
|
9
|
+
@passenger_config = if path.nil? || path.empty?
|
|
10
|
+
self.class.check_passenger_config(`which passenger-config`)
|
|
11
|
+
else
|
|
12
|
+
self.class.check_passenger_config!(path)
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
def do_kill(sig, pid, alive_sec, **params)
|
|
17
|
+
cmd = "#{passenger_config} detach-process #{pid}"
|
|
18
|
+
if sig == :KILL
|
|
19
|
+
logger.error "#{self} force to kill self (pid: #{pid}) alive: #{alive_sec} sec (trial #{kill_attempts})"
|
|
20
|
+
Process.kill sig, pid
|
|
21
|
+
return
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
return if @already_detached
|
|
25
|
+
|
|
26
|
+
logger.warn "#{self} run #{cmd.inspect} (pid: #{pid}) alive: #{alive_sec} sec (trial #{kill_attempts})"
|
|
27
|
+
@already_detached = true
|
|
28
|
+
|
|
29
|
+
Thread.new(cmd) do |command|
|
|
30
|
+
unless Kernel.system(command)
|
|
31
|
+
logger.warn "#{self} run #{command.inspect} failed: #{$?.inspect}"
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def self.check_passenger_config path
|
|
37
|
+
path.strip!
|
|
38
|
+
help = `#{path} detach-process --help 2> /dev/null`
|
|
39
|
+
return path if help['Remove an application process from the Phusion Passenger process pool']
|
|
40
|
+
rescue StandardError => e
|
|
41
|
+
nil
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def self.check_passenger_config! path
|
|
45
|
+
if path = check_passenger_config(path)
|
|
46
|
+
path
|
|
47
|
+
else
|
|
48
|
+
raise "Can't find passenger config at #{path.inspect}"
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
module WorkerKiller
|
|
2
|
+
module Killer
|
|
3
|
+
class Signal < ::WorkerKiller::Killer::Base
|
|
4
|
+
|
|
5
|
+
def do_kill(sig, pid, alive_sec, **params)
|
|
6
|
+
return if sig == @last_signal
|
|
7
|
+
|
|
8
|
+
@last_signal = sig
|
|
9
|
+
logger.warn "#{self} send SIG#{sig} (pid: #{pid}) alive: #{alive_sec} sec (trial #{kill_attempts})"
|
|
10
|
+
Process.kill sig, pid
|
|
11
|
+
end
|
|
12
|
+
|
|
13
|
+
end
|
|
14
|
+
end
|
|
15
|
+
end
|
|
16
|
+
|
|
@@ -4,16 +4,14 @@ module WorkerKiller
|
|
|
4
4
|
class MemoryLimiter
|
|
5
5
|
|
|
6
6
|
attr_reader :min, :max, :limit, :started_at, :check_cycle
|
|
7
|
-
attr_accessor :reaction
|
|
8
7
|
|
|
9
|
-
def initialize(min: (1024**3), max: (2 * (1024**3)), check_cycle: 16, verbose: false
|
|
8
|
+
def initialize(min: (1024**3), max: (2 * (1024**3)), check_cycle: 16, verbose: false)
|
|
10
9
|
@min = min
|
|
11
10
|
@max = max
|
|
12
11
|
@limit = @min + WorkerKiller.randomize(@max - @min + 1)
|
|
13
12
|
@check_cycle = check_cycle
|
|
14
13
|
@check_count = 0
|
|
15
14
|
@verbose = verbose
|
|
16
|
-
@reaction = block
|
|
17
15
|
end
|
|
18
16
|
|
|
19
17
|
def check
|
|
@@ -33,7 +31,6 @@ module WorkerKiller
|
|
|
33
31
|
return false if rss <= @limit
|
|
34
32
|
|
|
35
33
|
logger.warn "#{self}: worker (pid: #{Process.pid}) exceeds memory limit (#{rss} bytes > #{@limit} bytes)"
|
|
36
|
-
@reaction.call(self)
|
|
37
34
|
|
|
38
35
|
true
|
|
39
36
|
end
|
|
@@ -4,23 +4,22 @@ require 'worker_killer/count_limiter'
|
|
|
4
4
|
module WorkerKiller
|
|
5
5
|
class Middleware
|
|
6
6
|
|
|
7
|
-
attr_reader :limiter
|
|
7
|
+
attr_reader :limiter, :killer, :reaction
|
|
8
8
|
|
|
9
|
-
def initialize(app, klass:, reaction: nil, **opts)
|
|
9
|
+
def initialize(app, killer:, klass:, reaction: nil, **opts)
|
|
10
10
|
@app = app
|
|
11
|
+
@killer = killer
|
|
11
12
|
|
|
12
|
-
@
|
|
13
|
-
|
|
14
|
-
reaction ||= proc do |limiter|
|
|
15
|
-
WorkerKiller.kill_self(limiter.logger, limiter.started_at)
|
|
13
|
+
@reaction = reaction || proc do |l, k|
|
|
14
|
+
k.kill(l.started_at)
|
|
16
15
|
end
|
|
17
16
|
|
|
18
|
-
@limiter = klass.new(opts
|
|
17
|
+
@limiter = klass.new(opts)
|
|
19
18
|
end
|
|
20
19
|
|
|
21
20
|
def call(env)
|
|
22
21
|
response = @app.call(env)
|
|
23
|
-
|
|
22
|
+
reaction.call(limiter, killer) if limiter.check
|
|
24
23
|
response
|
|
25
24
|
end
|
|
26
25
|
|
data/spec/count_limiter_spec.rb
CHANGED
|
@@ -1,20 +1,15 @@
|
|
|
1
1
|
RSpec.describe WorkerKiller::CountLimiter do
|
|
2
|
-
let(:logger){ Logger.new(nil) }
|
|
3
|
-
|
|
4
2
|
subject{ described_class.new(options) }
|
|
5
3
|
let(:min){ rand(50..100) }
|
|
6
4
|
let(:max){ min + rand(100) }
|
|
7
|
-
let(:options){ { min: min, max: max } }
|
|
5
|
+
let(:options){ { min: min, max: max, verbose: true } }
|
|
8
6
|
|
|
9
7
|
it { is_expected.to have_attributes(min: min, max: max, limit: a_value_between(min, max), left: subject.limit) }
|
|
10
8
|
|
|
11
9
|
it 'expect not to react while less than limit' do
|
|
12
|
-
|
|
13
|
-
subject.
|
|
14
|
-
|
|
15
|
-
expect(subject.check).to be_falsey
|
|
16
|
-
end
|
|
17
|
-
end.not_to yield_control
|
|
10
|
+
(subject.limit - 1).times do
|
|
11
|
+
expect(subject.check).to be_falsey
|
|
12
|
+
end
|
|
18
13
|
end
|
|
19
14
|
|
|
20
15
|
it 'expect call reaction when check succeded' do
|
|
@@ -22,11 +17,8 @@ RSpec.describe WorkerKiller::CountLimiter do
|
|
|
22
17
|
expect(subject.check).to be_falsey
|
|
23
18
|
end
|
|
24
19
|
|
|
25
|
-
expect
|
|
26
|
-
|
|
27
|
-
expect(subject.check).to be_truthy
|
|
28
|
-
expect(subject.check).to be_truthy
|
|
29
|
-
end.to yield_control.exactly(2).times
|
|
20
|
+
expect(subject.check).to be_truthy
|
|
21
|
+
expect(subject.check).to be_truthy
|
|
30
22
|
end
|
|
31
23
|
end
|
|
32
24
|
|
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
RSpec.describe WorkerKiller::Killer::DelayedJob do
|
|
2
|
+
let(:config) do
|
|
3
|
+
WorkerKiller::Configuration.new.tap do |c|
|
|
4
|
+
c.quit_attempts = 2
|
|
5
|
+
c.term_attempts = 2
|
|
6
|
+
end
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
let(:killer){ described_class.new() }
|
|
10
|
+
let(:dj){ double }
|
|
11
|
+
|
|
12
|
+
describe '#kill' do
|
|
13
|
+
context 'with use_quit TRUE' do
|
|
14
|
+
around do |example|
|
|
15
|
+
prev = WorkerKiller.configuration
|
|
16
|
+
WorkerKiller.configuration = config
|
|
17
|
+
example.run
|
|
18
|
+
ensure
|
|
19
|
+
WorkerKiller.configuration = prev
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
it 'expect right signal order' do
|
|
23
|
+
expect(dj).to receive(:stop).exactly(4)
|
|
24
|
+
expect(Process).to receive(:kill).with(:TERM, anything).exactly(1).times
|
|
25
|
+
expect(Process).to receive(:kill).with(:KILL, anything).exactly(5).times
|
|
26
|
+
|
|
27
|
+
2.times { killer.kill(Time.now, dj: dj) } # 1 QUIT
|
|
28
|
+
2.times { killer.kill(Time.now, dj: dj) } # 1 TERM
|
|
29
|
+
5.times { killer.kill(Time.now, dj: dj) } # 5 KILL
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
RSpec.describe WorkerKiller::Killer::Passenger do
|
|
2
|
+
let(:config) do
|
|
3
|
+
WorkerKiller::Configuration.new.tap do |c|
|
|
4
|
+
c.quit_attempts = 2
|
|
5
|
+
c.term_attempts = 2
|
|
6
|
+
end
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
let(:killer){ described_class.new() }
|
|
10
|
+
|
|
11
|
+
describe '#kill' do
|
|
12
|
+
before do
|
|
13
|
+
allow(described_class).to receive(:check_passenger_config).and_return('custompath')
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
around do |example|
|
|
17
|
+
prev = WorkerKiller.configuration
|
|
18
|
+
WorkerKiller.configuration = config
|
|
19
|
+
example.run
|
|
20
|
+
ensure
|
|
21
|
+
WorkerKiller.configuration = prev
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
it 'expect right signal order' do
|
|
25
|
+
expect(Kernel).to receive(:system).with("custompath detach-process #{Process.pid}").and_return(true)
|
|
26
|
+
expect(Process).to receive(:kill).with(:KILL, anything).exactly(5).times
|
|
27
|
+
|
|
28
|
+
thread = killer.kill(Time.now)
|
|
29
|
+
thread.join
|
|
30
|
+
|
|
31
|
+
1.times { killer.kill(Time.now) } # 1 QUIT
|
|
32
|
+
2.times { killer.kill(Time.now) } # 1 TERM
|
|
33
|
+
5.times { killer.kill(Time.now) } # 5 KILL
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
describe '#check_passenger_config!' do
|
|
38
|
+
it do
|
|
39
|
+
expect{ described_class.check_passenger_config!('nonenone') }.to raise_error(/Can't find passenger/)
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
RSpec.describe WorkerKiller::Killer::Signal do
|
|
2
|
+
let(:config) do
|
|
3
|
+
WorkerKiller::Configuration.new.tap do |c|
|
|
4
|
+
c.quit_attempts = 2
|
|
5
|
+
c.term_attempts = 2
|
|
6
|
+
end
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
let(:killer){ described_class.new() }
|
|
10
|
+
|
|
11
|
+
describe '#kill' do
|
|
12
|
+
context 'with use_quit TRUE' do
|
|
13
|
+
around do |example|
|
|
14
|
+
prev = WorkerKiller.configuration
|
|
15
|
+
WorkerKiller.configuration = config
|
|
16
|
+
example.run
|
|
17
|
+
ensure
|
|
18
|
+
WorkerKiller.configuration = prev
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
it 'expect right signal order' do
|
|
22
|
+
expect(Process).to receive(:kill).with(:QUIT, anything).exactly(1).times
|
|
23
|
+
expect(Process).to receive(:kill).with(:TERM, anything).exactly(1).times
|
|
24
|
+
expect(Process).to receive(:kill).with(:KILL, anything).exactly(1).times
|
|
25
|
+
|
|
26
|
+
2.times { killer.kill(Time.now) } # 1 QUIT
|
|
27
|
+
2.times { killer.kill(Time.now) } # 1 TERM
|
|
28
|
+
5.times { killer.kill(Time.now) } # 1 KILL
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
|
data/spec/killer_spec.rb
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
RSpec.describe WorkerKiller::Killer::Base do
|
|
2
|
+
let(:config) do
|
|
3
|
+
WorkerKiller::Configuration.new.tap do |c|
|
|
4
|
+
c.quit_attempts = 2
|
|
5
|
+
c.term_attempts = 2
|
|
6
|
+
end
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
let(:killer){ described_class.new() }
|
|
10
|
+
|
|
11
|
+
describe '#kill' do
|
|
12
|
+
context 'with use_quit TRUE' do
|
|
13
|
+
around do |example|
|
|
14
|
+
prev = WorkerKiller.configuration
|
|
15
|
+
WorkerKiller.configuration = config
|
|
16
|
+
example.run
|
|
17
|
+
ensure
|
|
18
|
+
WorkerKiller.configuration = prev
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
it 'expect right signal order' do
|
|
22
|
+
expect(killer).to receive(:do_kill).with(:QUIT, anything, anything, anything).exactly(2).times
|
|
23
|
+
expect(killer).to receive(:do_kill).with(:TERM, anything, anything, anything).exactly(2).times
|
|
24
|
+
expect(killer).to receive(:do_kill).with(:KILL, anything, anything, anything).exactly(5).times
|
|
25
|
+
|
|
26
|
+
2.times { killer.kill(Time.now) } # 2 QUIT
|
|
27
|
+
2.times { killer.kill(Time.now) } # 2 TERM
|
|
28
|
+
5.times { killer.kill(Time.now) } # other - KILL
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
data/spec/memory_limiter_spec.rb
CHANGED
|
@@ -6,7 +6,7 @@ RSpec.describe WorkerKiller::MemoryLimiter do
|
|
|
6
6
|
let(:min){ rand(50..100) * mb }
|
|
7
7
|
let(:max){ min + rand(100) * mb }
|
|
8
8
|
let(:check_cycle){ 5 }
|
|
9
|
-
let(:options){ { min: min, max: max, check_cycle: check_cycle } }
|
|
9
|
+
let(:options){ { min: min, max: max, check_cycle: check_cycle, verbose: true} }
|
|
10
10
|
|
|
11
11
|
it { is_expected.to have_attributes(min: min, max: max, limit: a_value_between(min, max)) }
|
|
12
12
|
|
|
@@ -19,10 +19,7 @@ RSpec.describe WorkerKiller::MemoryLimiter do
|
|
|
19
19
|
it 'expect to skip check while less than cycle count' do
|
|
20
20
|
expect(GetProcessMem).not_to receive(:new)
|
|
21
21
|
|
|
22
|
-
|
|
23
|
-
subject.reaction = b.to_proc
|
|
24
|
-
skip_cycles(subject, check_cycle)
|
|
25
|
-
end.not_to yield_control
|
|
22
|
+
skip_cycles(subject, check_cycle)
|
|
26
23
|
end
|
|
27
24
|
|
|
28
25
|
it 'expect to skip check after cycle count reached' do
|
|
@@ -30,11 +27,8 @@ RSpec.describe WorkerKiller::MemoryLimiter do
|
|
|
30
27
|
expect(memory).to receive(:bytes).and_return(min - 1)
|
|
31
28
|
expect(GetProcessMem).to receive(:new).and_return(memory)
|
|
32
29
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
skip_cycles(subject, check_cycle)
|
|
36
|
-
expect(subject.check).to be_falsey
|
|
37
|
-
end.not_to yield_control
|
|
30
|
+
skip_cycles(subject, check_cycle)
|
|
31
|
+
expect(subject.check).to be_falsey
|
|
38
32
|
end
|
|
39
33
|
|
|
40
34
|
it 'expect call reaction when check succeded' do
|
|
@@ -42,11 +36,8 @@ RSpec.describe WorkerKiller::MemoryLimiter do
|
|
|
42
36
|
expect(memory).to receive(:bytes).and_return(subject.limit + 1)
|
|
43
37
|
expect(GetProcessMem).to receive(:new).and_return(memory)
|
|
44
38
|
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
skip_cycles(subject, check_cycle)
|
|
48
|
-
expect(subject.check).to be_truthy
|
|
49
|
-
end.to yield_control.exactly(1).times
|
|
39
|
+
skip_cycles(subject, check_cycle)
|
|
40
|
+
expect(subject.check).to be_truthy
|
|
50
41
|
end
|
|
51
42
|
end
|
|
52
43
|
|
data/spec/middleware_spec.rb
CHANGED
|
@@ -1,44 +1,79 @@
|
|
|
1
1
|
require 'securerandom'
|
|
2
2
|
|
|
3
3
|
RSpec.describe WorkerKiller::Middleware do
|
|
4
|
-
let(:
|
|
5
|
-
|
|
6
|
-
let(:
|
|
7
|
-
let(:reaction){ ->{} }
|
|
4
|
+
let(:app){ double(call: {}) }
|
|
5
|
+
let(:killer) { double }
|
|
6
|
+
let(:reaction){ double }
|
|
8
7
|
let(:anykey){ SecureRandom.hex(8) }
|
|
9
8
|
|
|
10
9
|
describe 'Custom class' do
|
|
11
10
|
let(:klass){ double }
|
|
12
|
-
let(:options){ { klass: klass, reaction: reaction, anykey: anykey } }
|
|
11
|
+
let(:options){ { killer: killer, klass: klass, reaction: reaction, anykey: anykey } }
|
|
13
12
|
subject{ described_class.new(app, options) }
|
|
14
13
|
|
|
15
14
|
it 'is expected to be initialized' do
|
|
16
15
|
expect(klass).to receive(:new).with(anykey: anykey).and_return(99)
|
|
17
16
|
expect(subject.limiter).to eq(99)
|
|
17
|
+
expect(subject.killer).to eq(killer)
|
|
18
18
|
end
|
|
19
19
|
end
|
|
20
20
|
|
|
21
21
|
describe WorkerKiller::Middleware::RequestsLimiter do
|
|
22
|
-
let(:options){ {
|
|
22
|
+
let(:options){ { killer: killer, min: 3, max: 3 } }
|
|
23
23
|
subject{ described_class.new(app, options) }
|
|
24
24
|
|
|
25
|
+
it 'is expected to be initialized without reaction' do
|
|
26
|
+
expect(WorkerKiller::CountLimiter).to receive(:new).with(min: 3, max: 3).and_call_original
|
|
27
|
+
expect(subject.limiter).to be_an(WorkerKiller::CountLimiter)
|
|
28
|
+
expect(subject.limiter.min).to eq(3)
|
|
29
|
+
expect(subject.limiter.max).to eq(3)
|
|
30
|
+
expect(killer).to receive(:kill).with(Time).twice
|
|
31
|
+
|
|
32
|
+
4.times do
|
|
33
|
+
subject.call({})
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
|
|
25
37
|
it 'is expected to be initialized with reaction' do
|
|
26
|
-
|
|
38
|
+
options[:reaction] = reaction
|
|
39
|
+
|
|
40
|
+
expect(WorkerKiller::CountLimiter).to receive(:new).with(min: 3, max: 3).and_call_original
|
|
27
41
|
expect(subject.limiter).to be_an(WorkerKiller::CountLimiter)
|
|
28
|
-
expect(subject.limiter.min).to eq(
|
|
29
|
-
expect(subject.limiter.
|
|
42
|
+
expect(subject.limiter.min).to eq(3)
|
|
43
|
+
expect(subject.limiter.max).to eq(3)
|
|
44
|
+
expect(reaction).to receive(:call).with(subject.limiter, killer).twice
|
|
45
|
+
|
|
46
|
+
4.times do
|
|
47
|
+
subject.call({})
|
|
48
|
+
end
|
|
30
49
|
end
|
|
31
50
|
end
|
|
32
51
|
|
|
33
52
|
describe WorkerKiller::Middleware::OOMLimiter do
|
|
34
|
-
let(:options){ {
|
|
53
|
+
let(:options){ { killer: killer, min: 2222, max: 2223 } }
|
|
35
54
|
subject{ described_class.new(app, options) }
|
|
36
55
|
|
|
56
|
+
it 'is expected to be initialized without reaction' do
|
|
57
|
+
expect(WorkerKiller::MemoryLimiter).to receive(:new).with(min: 2222, max: 2223).and_call_original
|
|
58
|
+
expect(subject.limiter).to be_an(WorkerKiller::MemoryLimiter)
|
|
59
|
+
expect(subject.limiter.min).to eq(2222)
|
|
60
|
+
expect(subject.limiter.max).to eq(2223)
|
|
61
|
+
expect(killer).to receive(:kill).with(subject.limiter.started_at).once
|
|
62
|
+
expect(subject.limiter).to receive(:check).and_return(true)
|
|
63
|
+
|
|
64
|
+
subject.call({})
|
|
65
|
+
end
|
|
66
|
+
|
|
37
67
|
it 'is expected to be initialized with reaction' do
|
|
38
|
-
|
|
68
|
+
options[:reaction] = reaction
|
|
69
|
+
expect(WorkerKiller::MemoryLimiter).to receive(:new).with(min: 2222, max: 2223).and_call_original
|
|
39
70
|
expect(subject.limiter).to be_an(WorkerKiller::MemoryLimiter)
|
|
40
71
|
expect(subject.limiter.min).to eq(2222)
|
|
41
|
-
expect(subject.limiter.
|
|
72
|
+
expect(subject.limiter.max).to eq(2223)
|
|
73
|
+
expect(reaction).to receive(:call).with(subject.limiter, killer)
|
|
74
|
+
expect(subject.limiter).to receive(:check).and_return(true)
|
|
75
|
+
|
|
76
|
+
subject.call({})
|
|
42
77
|
end
|
|
43
78
|
end
|
|
44
79
|
end
|
data/spec/spec_helper.rb
CHANGED
|
@@ -29,6 +29,7 @@ SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter.new([
|
|
|
29
29
|
SimpleCov.start
|
|
30
30
|
|
|
31
31
|
require 'worker_killer'
|
|
32
|
+
require 'worker_killer/delayed_job_plugin'
|
|
32
33
|
|
|
33
34
|
$root = File.join(File.dirname(__dir__), 'spec')
|
|
34
35
|
Dir[File.join(__dir__, 'support', '**', '*.rb')].sort.each {|f| require f }
|
data/spec/worker_killer_spec.rb
CHANGED
|
@@ -1,15 +1,5 @@
|
|
|
1
1
|
RSpec.describe WorkerKiller do
|
|
2
2
|
let(:logger){ Logger.new(nil) }
|
|
3
|
-
let(:config) do
|
|
4
|
-
WorkerKiller::Configuration.new.tap do |c|
|
|
5
|
-
c.quit_attempts = 2
|
|
6
|
-
c.kill_attempts = 2
|
|
7
|
-
end
|
|
8
|
-
end
|
|
9
|
-
|
|
10
|
-
before do
|
|
11
|
-
WorkerKiller.instance_variable_set('@kill_attempts', nil)
|
|
12
|
-
end
|
|
13
3
|
|
|
14
4
|
describe '#randomize' do
|
|
15
5
|
[1, 5, 25, 125, 5000, 10_000].each do |max|
|
|
@@ -22,68 +12,5 @@ RSpec.describe WorkerKiller do
|
|
|
22
12
|
end
|
|
23
13
|
end
|
|
24
14
|
end
|
|
25
|
-
|
|
26
|
-
describe '#kill_by_signal' do
|
|
27
|
-
[:QUIT, :TERM, :KILL].each do |sig|
|
|
28
|
-
it "must send #{sig} signal" do
|
|
29
|
-
pid = rand(1000)
|
|
30
|
-
expect(Process).to receive(:kill).with(sig, pid)
|
|
31
|
-
WorkerKiller.kill_by_signal(logger, 11111, sig, pid)
|
|
32
|
-
end
|
|
33
|
-
end
|
|
34
|
-
end
|
|
35
|
-
|
|
36
|
-
describe '#kill_by_passenger' do
|
|
37
|
-
it "must run passenger-config" do
|
|
38
|
-
pid = rand(1000)
|
|
39
|
-
path = "passenger-config-#{rand(1000)}"
|
|
40
|
-
expect(Kernel).to receive(:system).with("#{path} detach-process #{pid}").and_return(true)
|
|
41
|
-
|
|
42
|
-
thread = WorkerKiller.kill_by_passenger(logger, 11111, path, pid)
|
|
43
|
-
thread.join
|
|
44
|
-
end
|
|
45
|
-
end
|
|
46
|
-
|
|
47
|
-
describe '#kill_self' do
|
|
48
|
-
context 'with use_quit TRUE' do
|
|
49
|
-
around do |example|
|
|
50
|
-
prev = WorkerKiller.configuration
|
|
51
|
-
config.use_quit = true
|
|
52
|
-
WorkerKiller.configuration = config
|
|
53
|
-
example.run
|
|
54
|
-
ensure
|
|
55
|
-
WorkerKiller.configuration = prev
|
|
56
|
-
end
|
|
57
|
-
|
|
58
|
-
it 'expect right signal order' do
|
|
59
|
-
expect(WorkerKiller).to receive(:kill_by_signal).with(logger, anything, :QUIT, anything).exactly(2).times
|
|
60
|
-
expect(WorkerKiller).to receive(:kill_by_signal).with(logger, anything, :TERM, anything).exactly(2).times
|
|
61
|
-
expect(WorkerKiller).to receive(:kill_by_signal).with(logger, anything, :KILL, anything).exactly(5).times
|
|
62
|
-
|
|
63
|
-
2.times { WorkerKiller.kill_self(logger, Time.now) } # 2 QUIT
|
|
64
|
-
2.times { WorkerKiller.kill_self(logger, Time.now) } # 2 TERM
|
|
65
|
-
5.times { WorkerKiller.kill_self(logger, Time.now) } # other - KILL
|
|
66
|
-
end
|
|
67
|
-
end
|
|
68
|
-
|
|
69
|
-
context 'with use_quit FALSE' do
|
|
70
|
-
around do |example|
|
|
71
|
-
prev = WorkerKiller.configuration
|
|
72
|
-
config.use_quit = false
|
|
73
|
-
WorkerKiller.configuration = config
|
|
74
|
-
example.run
|
|
75
|
-
ensure
|
|
76
|
-
WorkerKiller.configuration = prev
|
|
77
|
-
end
|
|
78
|
-
|
|
79
|
-
it 'expect right signal order' do
|
|
80
|
-
expect(WorkerKiller).to receive(:kill_by_signal).with(logger, anything, :TERM, anything).exactly(2).times
|
|
81
|
-
expect(WorkerKiller).to receive(:kill_by_signal).with(logger, anything, :KILL, anything).exactly(5).times
|
|
82
|
-
|
|
83
|
-
2.times { WorkerKiller.kill_self(logger, Time.now) } # 2 TERM
|
|
84
|
-
5.times { WorkerKiller.kill_self(logger, Time.now) } # other - KILL
|
|
85
|
-
end
|
|
86
|
-
end
|
|
87
|
-
end
|
|
88
15
|
end
|
|
89
16
|
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: worker_killer
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.1.1.
|
|
4
|
+
version: 0.1.1.39838
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Samoilenko Yuri
|
|
8
8
|
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date: 2020-
|
|
11
|
+
date: 2020-10-28 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: get_process_mem
|
|
@@ -127,13 +127,23 @@ files:
|
|
|
127
127
|
- lib/worker_killer.rb
|
|
128
128
|
- lib/worker_killer/configuration.rb
|
|
129
129
|
- lib/worker_killer/count_limiter.rb
|
|
130
|
+
- lib/worker_killer/delayed_job_plugin.rb
|
|
131
|
+
- lib/worker_killer/killer.rb
|
|
132
|
+
- lib/worker_killer/killer/delayed_job.rb
|
|
133
|
+
- lib/worker_killer/killer/passenger.rb
|
|
134
|
+
- lib/worker_killer/killer/signal.rb
|
|
130
135
|
- lib/worker_killer/memory_limiter.rb
|
|
131
136
|
- lib/worker_killer/middleware.rb
|
|
132
137
|
- lib/worker_killer/version.rb
|
|
133
138
|
- spec/count_limiter_spec.rb
|
|
139
|
+
- spec/killer/delayed_job_spec.rb
|
|
140
|
+
- spec/killer/passenger_spec.rb
|
|
141
|
+
- spec/killer/signal_spec.rb
|
|
142
|
+
- spec/killer_spec.rb
|
|
134
143
|
- spec/memory_limiter_spec.rb
|
|
135
144
|
- spec/middleware_spec.rb
|
|
136
145
|
- spec/spec_helper.rb
|
|
146
|
+
- spec/support/logger.rb
|
|
137
147
|
- spec/worker_killer_spec.rb
|
|
138
148
|
homepage: https://github.com/RnD-Soft/worker_killer
|
|
139
149
|
licenses:
|
|
@@ -160,7 +170,12 @@ specification_version: 4
|
|
|
160
170
|
summary: Kill any workers by memory and request counts or take custom reaction
|
|
161
171
|
test_files:
|
|
162
172
|
- spec/middleware_spec.rb
|
|
173
|
+
- spec/killer_spec.rb
|
|
163
174
|
- spec/spec_helper.rb
|
|
175
|
+
- spec/killer/passenger_spec.rb
|
|
176
|
+
- spec/killer/signal_spec.rb
|
|
177
|
+
- spec/killer/delayed_job_spec.rb
|
|
164
178
|
- spec/memory_limiter_spec.rb
|
|
179
|
+
- spec/support/logger.rb
|
|
165
180
|
- spec/worker_killer_spec.rb
|
|
166
181
|
- spec/count_limiter_spec.rb
|