worker_killer 0.1.1.30345 → 0.1.1.39838
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/lib/worker_killer.rb +40 -39
- data/lib/worker_killer/configuration.rb +3 -35
- data/lib/worker_killer/count_limiter.rb +1 -4
- data/lib/worker_killer/delayed_job_plugin.rb +31 -0
- data/lib/worker_killer/killer.rb +38 -0
- data/lib/worker_killer/killer/delayed_job.rb +27 -0
- data/lib/worker_killer/killer/passenger.rb +55 -0
- data/lib/worker_killer/killer/signal.rb +16 -0
- data/lib/worker_killer/memory_limiter.rb +1 -4
- data/lib/worker_killer/middleware.rb +7 -8
- data/spec/count_limiter_spec.rb +6 -14
- data/spec/killer/delayed_job_spec.rb +34 -0
- data/spec/killer/passenger_spec.rb +43 -0
- data/spec/killer/signal_spec.rb +33 -0
- data/spec/killer_spec.rb +34 -0
- data/spec/memory_limiter_spec.rb +6 -15
- data/spec/middleware_spec.rb +47 -12
- data/spec/spec_helper.rb +1 -0
- data/spec/support/logger.rb +12 -0
- data/spec/worker_killer_spec.rb +0 -73
- metadata +17 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 92604c2f0b3b1fbd27a647f8c72556abbd2c6a38d3e57a120697118db1921b44
|
4
|
+
data.tar.gz: e22fb98b3d5dd0781e79eaba94213e7cca03c728f0f64ab0fe391b4f536de850
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: acabb632791e185856f983c44b3f7ae151e5798508e846c669b8699ec6bf06bd435c6a4a9fe551e94033e5a577ab252a8a3aaaffb09733ee52a27422faf0de0a
|
7
|
+
data.tar.gz: 253ae056f4b4d107458a85a897c3ad16a6ec532282bf7e9781fed06cfde943f177b23c6c3f96c3c513fe6f4a01ed92604c84eaca31bc64dca05ccfe2ba9c902e
|
data/lib/worker_killer.rb
CHANGED
@@ -3,6 +3,7 @@ require 'worker_killer/configuration'
|
|
3
3
|
require 'worker_killer/count_limiter'
|
4
4
|
require 'worker_killer/memory_limiter'
|
5
5
|
require 'worker_killer/middleware'
|
6
|
+
require 'worker_killer/killer'
|
6
7
|
|
7
8
|
module WorkerKiller
|
8
9
|
|
@@ -27,45 +28,45 @@ module WorkerKiller
|
|
27
28
|
# the process isn't killed after `configuration.quit_attempts` QUIT signals,
|
28
29
|
# send TERM signals until `configuration.kill_attempts`. Finally, send a KILL
|
29
30
|
# signal. A single signal is sent per request.
|
30
|
-
def self.kill_self(logger, start_time)
|
31
|
-
|
32
|
-
|
33
|
-
|
34
|
-
|
35
|
-
|
36
|
-
|
37
|
-
|
38
|
-
|
39
|
-
|
40
|
-
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
45
|
-
|
46
|
-
|
47
|
-
|
48
|
-
|
49
|
-
|
50
|
-
end
|
51
|
-
|
52
|
-
def self.kill_by_signal(logger, alive_sec, signal, pid)
|
53
|
-
|
54
|
-
|
55
|
-
end
|
56
|
-
|
57
|
-
def self.kill_by_passenger(logger, alive_sec, passenger, pid)
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
|
64
|
-
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
end
|
31
|
+
# def self.kill_self(logger, start_time)
|
32
|
+
# alive_sec = (Time.now - start_time).round
|
33
|
+
|
34
|
+
# @kill_attempts ||= 0
|
35
|
+
# @kill_attempts += 1
|
36
|
+
|
37
|
+
# if configuration.use_quit
|
38
|
+
# sig = :QUIT
|
39
|
+
# sig = :TERM if @kill_attempts > configuration.quit_attempts
|
40
|
+
# sig = :KILL if @kill_attempts > (configuration.quit_attempts + configuration.term_attempts)
|
41
|
+
# else
|
42
|
+
# sig = :TERM
|
43
|
+
# sig = :KILL if @kill_attempts > configuration.term_attempts
|
44
|
+
# end
|
45
|
+
|
46
|
+
# if sig == :QUIT && configuration.passenger?
|
47
|
+
# kill_by_passenger(logger, alive_sec, configuration.passenger_config, Process.pid)
|
48
|
+
# else
|
49
|
+
# kill_by_signal(logger, alive_sec, sig, Process.pid)
|
50
|
+
# end
|
51
|
+
# end
|
52
|
+
|
53
|
+
# def self.kill_by_signal(logger, alive_sec, signal, pid)
|
54
|
+
# logger.warn "#{self} send SIG#{signal} (pid: #{pid}) alive: #{alive_sec} sec (trial #{@kill_attempts})"
|
55
|
+
# Process.kill signal, pid
|
56
|
+
# end
|
57
|
+
|
58
|
+
# def self.kill_by_passenger(logger, alive_sec, passenger, pid)
|
59
|
+
# return if @already_detached
|
60
|
+
# @already_detached = true
|
61
|
+
|
62
|
+
# cmd = "#{passenger} detach-process #{pid}"
|
63
|
+
# logger.warn "#{self} run #{cmd.inspect} (pid: #{pid}) alive: #{alive_sec} sec (trial #{@kill_attempts})"
|
64
|
+
# Thread.new(cmd) do |command|
|
65
|
+
# unless Kernel.system(command)
|
66
|
+
# logger.warn "#{self} run #{cmd.inspect} failed: #{$?.inspect}"
|
67
|
+
# end
|
68
|
+
# end
|
69
|
+
# end
|
69
70
|
|
70
71
|
end
|
71
72
|
|
@@ -4,47 +4,15 @@ module WorkerKiller
|
|
4
4
|
# Methods for configuring WorkerKiller
|
5
5
|
class Configuration
|
6
6
|
|
7
|
-
attr_accessor :logger, :quit_attempts, :
|
7
|
+
attr_accessor :logger, :quit_attempts, :term_attempts
|
8
8
|
|
9
9
|
# Override defaults for configuration
|
10
|
-
def initialize(quit_attempts:
|
10
|
+
def initialize(quit_attempts: 10, term_attempts: 50)
|
11
11
|
@quit_attempts = quit_attempts
|
12
|
-
@
|
13
|
-
@use_quit = use_quit
|
14
|
-
@passenger_config = check_passenger(passenger_config)
|
12
|
+
@term_attempts = term_attempts
|
15
13
|
@logger = Logger.new(STDOUT, level: Logger::INFO, progname: 'WorkerKiller')
|
16
14
|
end
|
17
15
|
|
18
|
-
def passenger?
|
19
|
-
!@passenger_config.nil?
|
20
|
-
end
|
21
|
-
|
22
|
-
def check_passenger provided_path
|
23
|
-
if provided_path.nil? || provided_path.empty?
|
24
|
-
return check_passenger_config(`which passenger-config`)
|
25
|
-
else
|
26
|
-
return check_passenger_config!(provided_path)
|
27
|
-
end
|
28
|
-
end
|
29
|
-
|
30
|
-
def check_passenger_config path
|
31
|
-
path.strip!
|
32
|
-
help = `#{path} detach-process --help 2> /dev/null`
|
33
|
-
if help['Remove an application process from the Phusion Passenger process pool']
|
34
|
-
return path
|
35
|
-
end
|
36
|
-
rescue StandardError => e
|
37
|
-
return nil
|
38
|
-
end
|
39
|
-
|
40
|
-
def check_passenger_config! path
|
41
|
-
if path = check_passenger_config(path)
|
42
|
-
return path
|
43
|
-
else
|
44
|
-
raise "Can't find passenger config at #{path.inspect}"
|
45
|
-
end
|
46
|
-
end
|
47
|
-
|
48
16
|
end
|
49
17
|
end
|
50
18
|
|
@@ -2,15 +2,13 @@ module WorkerKiller
|
|
2
2
|
class CountLimiter
|
3
3
|
|
4
4
|
attr_reader :min, :max, :left, :limit, :started_at
|
5
|
-
attr_accessor :reaction
|
6
5
|
|
7
|
-
def initialize(min: 3072, max: 4096, verbose: false
|
6
|
+
def initialize(min: 3072, max: 4096, verbose: false)
|
8
7
|
@min = min
|
9
8
|
@max = max
|
10
9
|
@limit = @min + WorkerKiller.randomize(@max - @min + 1)
|
11
10
|
@left = @limit
|
12
11
|
@verbose = verbose
|
13
|
-
@reaction = block
|
14
12
|
end
|
15
13
|
|
16
14
|
def check
|
@@ -25,7 +23,6 @@ module WorkerKiller
|
|
25
23
|
return false if (@left -= 1) > 0
|
26
24
|
|
27
25
|
logger.warn "#{self}: worker (pid: #{Process.pid}) exceeds max number of requests (limit: #{@limit})"
|
28
|
-
@reaction.call(self)
|
29
26
|
|
30
27
|
true
|
31
28
|
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
require 'worker_killer/memory_limiter'
|
2
|
+
require 'worker_killer/count_limiter'
|
3
|
+
|
4
|
+
module WorkerKiller
|
5
|
+
class DelayedJobPlugin
|
6
|
+
|
7
|
+
attr_reader :limiter, :killer, :reaction
|
8
|
+
|
9
|
+
def initialize(klass:, killer:, reaction: nil, **opts)
|
10
|
+
@killer = killer
|
11
|
+
|
12
|
+
@reaction = reaction || proc do |l, k, dj|
|
13
|
+
k.kill(l.started_at, dj: dj)
|
14
|
+
end
|
15
|
+
|
16
|
+
@limiter = klass.new(opts)
|
17
|
+
end
|
18
|
+
|
19
|
+
def new(*_args)
|
20
|
+
configure_lifecycle(Delayed::Worker.lifecycle)
|
21
|
+
end
|
22
|
+
|
23
|
+
def configure_lifecycle(lifecycle)
|
24
|
+
lifecycle.after(:perform) do |worker, *_args|
|
25
|
+
reaction.call(limiter, killer, worker) if limiter.check
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
@@ -0,0 +1,38 @@
|
|
1
|
+
module WorkerKiller
|
2
|
+
module Killer
|
3
|
+
class Base
|
4
|
+
|
5
|
+
attr_accessor :config, :kill_attempts, :logger
|
6
|
+
|
7
|
+
def initialize(logger: WorkerKiller.configuration.logger, **_kwargs)
|
8
|
+
@logger = logger
|
9
|
+
@config = WorkerKiller.configuration
|
10
|
+
@kill_attempts = 0
|
11
|
+
end
|
12
|
+
|
13
|
+
def kill(start_time, **params)
|
14
|
+
alive_sec = (Time.now - start_time).round
|
15
|
+
|
16
|
+
@kill_attempts += 1
|
17
|
+
|
18
|
+
sig = :QUIT
|
19
|
+
sig = :TERM if kill_attempts > config.quit_attempts
|
20
|
+
sig = :KILL if kill_attempts > (config.quit_attempts + config.term_attempts)
|
21
|
+
|
22
|
+
do_kill(sig, Process.pid, alive_sec, **params)
|
23
|
+
end
|
24
|
+
|
25
|
+
# :nocov:
|
26
|
+
def do_kill(*_args)
|
27
|
+
raise 'Not Implemented'
|
28
|
+
end
|
29
|
+
# :nocov:
|
30
|
+
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
35
|
+
require_relative 'killer/signal'
|
36
|
+
require_relative 'killer/passenger'
|
37
|
+
require_relative 'killer/delayed_job'
|
38
|
+
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module WorkerKiller
|
2
|
+
module Killer
|
3
|
+
class DelayedJob < ::WorkerKiller::Killer::Base
|
4
|
+
|
5
|
+
def do_kill(sig, pid, alive_sec, dj:, **_params)
|
6
|
+
if sig == :KILL
|
7
|
+
logger.error "#{self} force to #{sig} self (pid: #{pid}) alive: #{alive_sec} sec (trial #{kill_attempts})"
|
8
|
+
Process.kill sig, pid
|
9
|
+
return
|
10
|
+
end
|
11
|
+
|
12
|
+
dj.stop
|
13
|
+
|
14
|
+
return if sig != :TERM
|
15
|
+
|
16
|
+
if @termination
|
17
|
+
logger.warn "#{self} force to #{sig} self (pid: #{pid}) alive: #{alive_sec} sec (trial #{kill_attempts})"
|
18
|
+
Process.kill sig, pid
|
19
|
+
else
|
20
|
+
@termination = true
|
21
|
+
end
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
@@ -0,0 +1,55 @@
|
|
1
|
+
module WorkerKiller
|
2
|
+
module Killer
|
3
|
+
class Passenger < ::WorkerKiller::Killer::Base
|
4
|
+
|
5
|
+
attr_reader :passenger_config
|
6
|
+
|
7
|
+
def initialize path: nil, **kwrags
|
8
|
+
super
|
9
|
+
@passenger_config = if path.nil? || path.empty?
|
10
|
+
self.class.check_passenger_config(`which passenger-config`)
|
11
|
+
else
|
12
|
+
self.class.check_passenger_config!(path)
|
13
|
+
end
|
14
|
+
end
|
15
|
+
|
16
|
+
def do_kill(sig, pid, alive_sec, **params)
|
17
|
+
cmd = "#{passenger_config} detach-process #{pid}"
|
18
|
+
if sig == :KILL
|
19
|
+
logger.error "#{self} force to kill self (pid: #{pid}) alive: #{alive_sec} sec (trial #{kill_attempts})"
|
20
|
+
Process.kill sig, pid
|
21
|
+
return
|
22
|
+
end
|
23
|
+
|
24
|
+
return if @already_detached
|
25
|
+
|
26
|
+
logger.warn "#{self} run #{cmd.inspect} (pid: #{pid}) alive: #{alive_sec} sec (trial #{kill_attempts})"
|
27
|
+
@already_detached = true
|
28
|
+
|
29
|
+
Thread.new(cmd) do |command|
|
30
|
+
unless Kernel.system(command)
|
31
|
+
logger.warn "#{self} run #{command.inspect} failed: #{$?.inspect}"
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.check_passenger_config path
|
37
|
+
path.strip!
|
38
|
+
help = `#{path} detach-process --help 2> /dev/null`
|
39
|
+
return path if help['Remove an application process from the Phusion Passenger process pool']
|
40
|
+
rescue StandardError => e
|
41
|
+
nil
|
42
|
+
end
|
43
|
+
|
44
|
+
def self.check_passenger_config! path
|
45
|
+
if path = check_passenger_config(path)
|
46
|
+
path
|
47
|
+
else
|
48
|
+
raise "Can't find passenger config at #{path.inspect}"
|
49
|
+
end
|
50
|
+
end
|
51
|
+
|
52
|
+
end
|
53
|
+
end
|
54
|
+
end
|
55
|
+
|
@@ -0,0 +1,16 @@
|
|
1
|
+
module WorkerKiller
|
2
|
+
module Killer
|
3
|
+
class Signal < ::WorkerKiller::Killer::Base
|
4
|
+
|
5
|
+
def do_kill(sig, pid, alive_sec, **params)
|
6
|
+
return if sig == @last_signal
|
7
|
+
|
8
|
+
@last_signal = sig
|
9
|
+
logger.warn "#{self} send SIG#{sig} (pid: #{pid}) alive: #{alive_sec} sec (trial #{kill_attempts})"
|
10
|
+
Process.kill sig, pid
|
11
|
+
end
|
12
|
+
|
13
|
+
end
|
14
|
+
end
|
15
|
+
end
|
16
|
+
|
@@ -4,16 +4,14 @@ module WorkerKiller
|
|
4
4
|
class MemoryLimiter
|
5
5
|
|
6
6
|
attr_reader :min, :max, :limit, :started_at, :check_cycle
|
7
|
-
attr_accessor :reaction
|
8
7
|
|
9
|
-
def initialize(min: (1024**3), max: (2 * (1024**3)), check_cycle: 16, verbose: false
|
8
|
+
def initialize(min: (1024**3), max: (2 * (1024**3)), check_cycle: 16, verbose: false)
|
10
9
|
@min = min
|
11
10
|
@max = max
|
12
11
|
@limit = @min + WorkerKiller.randomize(@max - @min + 1)
|
13
12
|
@check_cycle = check_cycle
|
14
13
|
@check_count = 0
|
15
14
|
@verbose = verbose
|
16
|
-
@reaction = block
|
17
15
|
end
|
18
16
|
|
19
17
|
def check
|
@@ -33,7 +31,6 @@ module WorkerKiller
|
|
33
31
|
return false if rss <= @limit
|
34
32
|
|
35
33
|
logger.warn "#{self}: worker (pid: #{Process.pid}) exceeds memory limit (#{rss} bytes > #{@limit} bytes)"
|
36
|
-
@reaction.call(self)
|
37
34
|
|
38
35
|
true
|
39
36
|
end
|
@@ -4,23 +4,22 @@ require 'worker_killer/count_limiter'
|
|
4
4
|
module WorkerKiller
|
5
5
|
class Middleware
|
6
6
|
|
7
|
-
attr_reader :limiter
|
7
|
+
attr_reader :limiter, :killer, :reaction
|
8
8
|
|
9
|
-
def initialize(app, klass:, reaction: nil, **opts)
|
9
|
+
def initialize(app, killer:, klass:, reaction: nil, **opts)
|
10
10
|
@app = app
|
11
|
+
@killer = killer
|
11
12
|
|
12
|
-
@
|
13
|
-
|
14
|
-
reaction ||= proc do |limiter|
|
15
|
-
WorkerKiller.kill_self(limiter.logger, limiter.started_at)
|
13
|
+
@reaction = reaction || proc do |l, k|
|
14
|
+
k.kill(l.started_at)
|
16
15
|
end
|
17
16
|
|
18
|
-
@limiter = klass.new(opts
|
17
|
+
@limiter = klass.new(opts)
|
19
18
|
end
|
20
19
|
|
21
20
|
def call(env)
|
22
21
|
response = @app.call(env)
|
23
|
-
|
22
|
+
reaction.call(limiter, killer) if limiter.check
|
24
23
|
response
|
25
24
|
end
|
26
25
|
|
data/spec/count_limiter_spec.rb
CHANGED
@@ -1,20 +1,15 @@
|
|
1
1
|
RSpec.describe WorkerKiller::CountLimiter do
|
2
|
-
let(:logger){ Logger.new(nil) }
|
3
|
-
|
4
2
|
subject{ described_class.new(options) }
|
5
3
|
let(:min){ rand(50..100) }
|
6
4
|
let(:max){ min + rand(100) }
|
7
|
-
let(:options){ { min: min, max: max } }
|
5
|
+
let(:options){ { min: min, max: max, verbose: true } }
|
8
6
|
|
9
7
|
it { is_expected.to have_attributes(min: min, max: max, limit: a_value_between(min, max), left: subject.limit) }
|
10
8
|
|
11
9
|
it 'expect not to react while less than limit' do
|
12
|
-
|
13
|
-
subject.
|
14
|
-
|
15
|
-
expect(subject.check).to be_falsey
|
16
|
-
end
|
17
|
-
end.not_to yield_control
|
10
|
+
(subject.limit - 1).times do
|
11
|
+
expect(subject.check).to be_falsey
|
12
|
+
end
|
18
13
|
end
|
19
14
|
|
20
15
|
it 'expect call reaction when check succeded' do
|
@@ -22,11 +17,8 @@ RSpec.describe WorkerKiller::CountLimiter do
|
|
22
17
|
expect(subject.check).to be_falsey
|
23
18
|
end
|
24
19
|
|
25
|
-
expect
|
26
|
-
|
27
|
-
expect(subject.check).to be_truthy
|
28
|
-
expect(subject.check).to be_truthy
|
29
|
-
end.to yield_control.exactly(2).times
|
20
|
+
expect(subject.check).to be_truthy
|
21
|
+
expect(subject.check).to be_truthy
|
30
22
|
end
|
31
23
|
end
|
32
24
|
|
@@ -0,0 +1,34 @@
|
|
1
|
+
RSpec.describe WorkerKiller::Killer::DelayedJob do
|
2
|
+
let(:config) do
|
3
|
+
WorkerKiller::Configuration.new.tap do |c|
|
4
|
+
c.quit_attempts = 2
|
5
|
+
c.term_attempts = 2
|
6
|
+
end
|
7
|
+
end
|
8
|
+
|
9
|
+
let(:killer){ described_class.new() }
|
10
|
+
let(:dj){ double }
|
11
|
+
|
12
|
+
describe '#kill' do
|
13
|
+
context 'with use_quit TRUE' do
|
14
|
+
around do |example|
|
15
|
+
prev = WorkerKiller.configuration
|
16
|
+
WorkerKiller.configuration = config
|
17
|
+
example.run
|
18
|
+
ensure
|
19
|
+
WorkerKiller.configuration = prev
|
20
|
+
end
|
21
|
+
|
22
|
+
it 'expect right signal order' do
|
23
|
+
expect(dj).to receive(:stop).exactly(4)
|
24
|
+
expect(Process).to receive(:kill).with(:TERM, anything).exactly(1).times
|
25
|
+
expect(Process).to receive(:kill).with(:KILL, anything).exactly(5).times
|
26
|
+
|
27
|
+
2.times { killer.kill(Time.now, dj: dj) } # 1 QUIT
|
28
|
+
2.times { killer.kill(Time.now, dj: dj) } # 1 TERM
|
29
|
+
5.times { killer.kill(Time.now, dj: dj) } # 5 KILL
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
@@ -0,0 +1,43 @@
|
|
1
|
+
RSpec.describe WorkerKiller::Killer::Passenger do
|
2
|
+
let(:config) do
|
3
|
+
WorkerKiller::Configuration.new.tap do |c|
|
4
|
+
c.quit_attempts = 2
|
5
|
+
c.term_attempts = 2
|
6
|
+
end
|
7
|
+
end
|
8
|
+
|
9
|
+
let(:killer){ described_class.new() }
|
10
|
+
|
11
|
+
describe '#kill' do
|
12
|
+
before do
|
13
|
+
allow(described_class).to receive(:check_passenger_config).and_return('custompath')
|
14
|
+
end
|
15
|
+
|
16
|
+
around do |example|
|
17
|
+
prev = WorkerKiller.configuration
|
18
|
+
WorkerKiller.configuration = config
|
19
|
+
example.run
|
20
|
+
ensure
|
21
|
+
WorkerKiller.configuration = prev
|
22
|
+
end
|
23
|
+
|
24
|
+
it 'expect right signal order' do
|
25
|
+
expect(Kernel).to receive(:system).with("custompath detach-process #{Process.pid}").and_return(true)
|
26
|
+
expect(Process).to receive(:kill).with(:KILL, anything).exactly(5).times
|
27
|
+
|
28
|
+
thread = killer.kill(Time.now)
|
29
|
+
thread.join
|
30
|
+
|
31
|
+
1.times { killer.kill(Time.now) } # 1 QUIT
|
32
|
+
2.times { killer.kill(Time.now) } # 1 TERM
|
33
|
+
5.times { killer.kill(Time.now) } # 5 KILL
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
describe '#check_passenger_config!' do
|
38
|
+
it do
|
39
|
+
expect{ described_class.check_passenger_config!('nonenone') }.to raise_error(/Can't find passenger/)
|
40
|
+
end
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
@@ -0,0 +1,33 @@
|
|
1
|
+
RSpec.describe WorkerKiller::Killer::Signal do
|
2
|
+
let(:config) do
|
3
|
+
WorkerKiller::Configuration.new.tap do |c|
|
4
|
+
c.quit_attempts = 2
|
5
|
+
c.term_attempts = 2
|
6
|
+
end
|
7
|
+
end
|
8
|
+
|
9
|
+
let(:killer){ described_class.new() }
|
10
|
+
|
11
|
+
describe '#kill' do
|
12
|
+
context 'with use_quit TRUE' do
|
13
|
+
around do |example|
|
14
|
+
prev = WorkerKiller.configuration
|
15
|
+
WorkerKiller.configuration = config
|
16
|
+
example.run
|
17
|
+
ensure
|
18
|
+
WorkerKiller.configuration = prev
|
19
|
+
end
|
20
|
+
|
21
|
+
it 'expect right signal order' do
|
22
|
+
expect(Process).to receive(:kill).with(:QUIT, anything).exactly(1).times
|
23
|
+
expect(Process).to receive(:kill).with(:TERM, anything).exactly(1).times
|
24
|
+
expect(Process).to receive(:kill).with(:KILL, anything).exactly(1).times
|
25
|
+
|
26
|
+
2.times { killer.kill(Time.now) } # 1 QUIT
|
27
|
+
2.times { killer.kill(Time.now) } # 1 TERM
|
28
|
+
5.times { killer.kill(Time.now) } # 1 KILL
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
33
|
+
|
data/spec/killer_spec.rb
ADDED
@@ -0,0 +1,34 @@
|
|
1
|
+
RSpec.describe WorkerKiller::Killer::Base do
|
2
|
+
let(:config) do
|
3
|
+
WorkerKiller::Configuration.new.tap do |c|
|
4
|
+
c.quit_attempts = 2
|
5
|
+
c.term_attempts = 2
|
6
|
+
end
|
7
|
+
end
|
8
|
+
|
9
|
+
let(:killer){ described_class.new() }
|
10
|
+
|
11
|
+
describe '#kill' do
|
12
|
+
context 'with use_quit TRUE' do
|
13
|
+
around do |example|
|
14
|
+
prev = WorkerKiller.configuration
|
15
|
+
WorkerKiller.configuration = config
|
16
|
+
example.run
|
17
|
+
ensure
|
18
|
+
WorkerKiller.configuration = prev
|
19
|
+
end
|
20
|
+
|
21
|
+
it 'expect right signal order' do
|
22
|
+
expect(killer).to receive(:do_kill).with(:QUIT, anything, anything, anything).exactly(2).times
|
23
|
+
expect(killer).to receive(:do_kill).with(:TERM, anything, anything, anything).exactly(2).times
|
24
|
+
expect(killer).to receive(:do_kill).with(:KILL, anything, anything, anything).exactly(5).times
|
25
|
+
|
26
|
+
2.times { killer.kill(Time.now) } # 2 QUIT
|
27
|
+
2.times { killer.kill(Time.now) } # 2 TERM
|
28
|
+
5.times { killer.kill(Time.now) } # other - KILL
|
29
|
+
end
|
30
|
+
end
|
31
|
+
|
32
|
+
end
|
33
|
+
end
|
34
|
+
|
data/spec/memory_limiter_spec.rb
CHANGED
@@ -6,7 +6,7 @@ RSpec.describe WorkerKiller::MemoryLimiter do
|
|
6
6
|
let(:min){ rand(50..100) * mb }
|
7
7
|
let(:max){ min + rand(100) * mb }
|
8
8
|
let(:check_cycle){ 5 }
|
9
|
-
let(:options){ { min: min, max: max, check_cycle: check_cycle } }
|
9
|
+
let(:options){ { min: min, max: max, check_cycle: check_cycle, verbose: true} }
|
10
10
|
|
11
11
|
it { is_expected.to have_attributes(min: min, max: max, limit: a_value_between(min, max)) }
|
12
12
|
|
@@ -19,10 +19,7 @@ RSpec.describe WorkerKiller::MemoryLimiter do
|
|
19
19
|
it 'expect to skip check while less than cycle count' do
|
20
20
|
expect(GetProcessMem).not_to receive(:new)
|
21
21
|
|
22
|
-
|
23
|
-
subject.reaction = b.to_proc
|
24
|
-
skip_cycles(subject, check_cycle)
|
25
|
-
end.not_to yield_control
|
22
|
+
skip_cycles(subject, check_cycle)
|
26
23
|
end
|
27
24
|
|
28
25
|
it 'expect to skip check after cycle count reached' do
|
@@ -30,11 +27,8 @@ RSpec.describe WorkerKiller::MemoryLimiter do
|
|
30
27
|
expect(memory).to receive(:bytes).and_return(min - 1)
|
31
28
|
expect(GetProcessMem).to receive(:new).and_return(memory)
|
32
29
|
|
33
|
-
|
34
|
-
|
35
|
-
skip_cycles(subject, check_cycle)
|
36
|
-
expect(subject.check).to be_falsey
|
37
|
-
end.not_to yield_control
|
30
|
+
skip_cycles(subject, check_cycle)
|
31
|
+
expect(subject.check).to be_falsey
|
38
32
|
end
|
39
33
|
|
40
34
|
it 'expect call reaction when check succeded' do
|
@@ -42,11 +36,8 @@ RSpec.describe WorkerKiller::MemoryLimiter do
|
|
42
36
|
expect(memory).to receive(:bytes).and_return(subject.limit + 1)
|
43
37
|
expect(GetProcessMem).to receive(:new).and_return(memory)
|
44
38
|
|
45
|
-
|
46
|
-
|
47
|
-
skip_cycles(subject, check_cycle)
|
48
|
-
expect(subject.check).to be_truthy
|
49
|
-
end.to yield_control.exactly(1).times
|
39
|
+
skip_cycles(subject, check_cycle)
|
40
|
+
expect(subject.check).to be_truthy
|
50
41
|
end
|
51
42
|
end
|
52
43
|
|
data/spec/middleware_spec.rb
CHANGED
@@ -1,44 +1,79 @@
|
|
1
1
|
require 'securerandom'
|
2
2
|
|
3
3
|
RSpec.describe WorkerKiller::Middleware do
|
4
|
-
let(:
|
5
|
-
|
6
|
-
let(:
|
7
|
-
let(:reaction){ ->{} }
|
4
|
+
let(:app){ double(call: {}) }
|
5
|
+
let(:killer) { double }
|
6
|
+
let(:reaction){ double }
|
8
7
|
let(:anykey){ SecureRandom.hex(8) }
|
9
8
|
|
10
9
|
describe 'Custom class' do
|
11
10
|
let(:klass){ double }
|
12
|
-
let(:options){ { klass: klass, reaction: reaction, anykey: anykey } }
|
11
|
+
let(:options){ { killer: killer, klass: klass, reaction: reaction, anykey: anykey } }
|
13
12
|
subject{ described_class.new(app, options) }
|
14
13
|
|
15
14
|
it 'is expected to be initialized' do
|
16
15
|
expect(klass).to receive(:new).with(anykey: anykey).and_return(99)
|
17
16
|
expect(subject.limiter).to eq(99)
|
17
|
+
expect(subject.killer).to eq(killer)
|
18
18
|
end
|
19
19
|
end
|
20
20
|
|
21
21
|
describe WorkerKiller::Middleware::RequestsLimiter do
|
22
|
-
let(:options){ {
|
22
|
+
let(:options){ { killer: killer, min: 3, max: 3 } }
|
23
23
|
subject{ described_class.new(app, options) }
|
24
24
|
|
25
|
+
it 'is expected to be initialized without reaction' do
|
26
|
+
expect(WorkerKiller::CountLimiter).to receive(:new).with(min: 3, max: 3).and_call_original
|
27
|
+
expect(subject.limiter).to be_an(WorkerKiller::CountLimiter)
|
28
|
+
expect(subject.limiter.min).to eq(3)
|
29
|
+
expect(subject.limiter.max).to eq(3)
|
30
|
+
expect(killer).to receive(:kill).with(Time).twice
|
31
|
+
|
32
|
+
4.times do
|
33
|
+
subject.call({})
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
25
37
|
it 'is expected to be initialized with reaction' do
|
26
|
-
|
38
|
+
options[:reaction] = reaction
|
39
|
+
|
40
|
+
expect(WorkerKiller::CountLimiter).to receive(:new).with(min: 3, max: 3).and_call_original
|
27
41
|
expect(subject.limiter).to be_an(WorkerKiller::CountLimiter)
|
28
|
-
expect(subject.limiter.min).to eq(
|
29
|
-
expect(subject.limiter.
|
42
|
+
expect(subject.limiter.min).to eq(3)
|
43
|
+
expect(subject.limiter.max).to eq(3)
|
44
|
+
expect(reaction).to receive(:call).with(subject.limiter, killer).twice
|
45
|
+
|
46
|
+
4.times do
|
47
|
+
subject.call({})
|
48
|
+
end
|
30
49
|
end
|
31
50
|
end
|
32
51
|
|
33
52
|
describe WorkerKiller::Middleware::OOMLimiter do
|
34
|
-
let(:options){ {
|
53
|
+
let(:options){ { killer: killer, min: 2222, max: 2223 } }
|
35
54
|
subject{ described_class.new(app, options) }
|
36
55
|
|
56
|
+
it 'is expected to be initialized without reaction' do
|
57
|
+
expect(WorkerKiller::MemoryLimiter).to receive(:new).with(min: 2222, max: 2223).and_call_original
|
58
|
+
expect(subject.limiter).to be_an(WorkerKiller::MemoryLimiter)
|
59
|
+
expect(subject.limiter.min).to eq(2222)
|
60
|
+
expect(subject.limiter.max).to eq(2223)
|
61
|
+
expect(killer).to receive(:kill).with(subject.limiter.started_at).once
|
62
|
+
expect(subject.limiter).to receive(:check).and_return(true)
|
63
|
+
|
64
|
+
subject.call({})
|
65
|
+
end
|
66
|
+
|
37
67
|
it 'is expected to be initialized with reaction' do
|
38
|
-
|
68
|
+
options[:reaction] = reaction
|
69
|
+
expect(WorkerKiller::MemoryLimiter).to receive(:new).with(min: 2222, max: 2223).and_call_original
|
39
70
|
expect(subject.limiter).to be_an(WorkerKiller::MemoryLimiter)
|
40
71
|
expect(subject.limiter.min).to eq(2222)
|
41
|
-
expect(subject.limiter.
|
72
|
+
expect(subject.limiter.max).to eq(2223)
|
73
|
+
expect(reaction).to receive(:call).with(subject.limiter, killer)
|
74
|
+
expect(subject.limiter).to receive(:check).and_return(true)
|
75
|
+
|
76
|
+
subject.call({})
|
42
77
|
end
|
43
78
|
end
|
44
79
|
end
|
data/spec/spec_helper.rb
CHANGED
@@ -29,6 +29,7 @@ SimpleCov.formatter = SimpleCov::Formatter::MultiFormatter.new([
|
|
29
29
|
SimpleCov.start
|
30
30
|
|
31
31
|
require 'worker_killer'
|
32
|
+
require 'worker_killer/delayed_job_plugin'
|
32
33
|
|
33
34
|
$root = File.join(File.dirname(__dir__), 'spec')
|
34
35
|
Dir[File.join(__dir__, 'support', '**', '*.rb')].sort.each {|f| require f }
|
data/spec/worker_killer_spec.rb
CHANGED
@@ -1,15 +1,5 @@
|
|
1
1
|
RSpec.describe WorkerKiller do
|
2
2
|
let(:logger){ Logger.new(nil) }
|
3
|
-
let(:config) do
|
4
|
-
WorkerKiller::Configuration.new.tap do |c|
|
5
|
-
c.quit_attempts = 2
|
6
|
-
c.kill_attempts = 2
|
7
|
-
end
|
8
|
-
end
|
9
|
-
|
10
|
-
before do
|
11
|
-
WorkerKiller.instance_variable_set('@kill_attempts', nil)
|
12
|
-
end
|
13
3
|
|
14
4
|
describe '#randomize' do
|
15
5
|
[1, 5, 25, 125, 5000, 10_000].each do |max|
|
@@ -22,68 +12,5 @@ RSpec.describe WorkerKiller do
|
|
22
12
|
end
|
23
13
|
end
|
24
14
|
end
|
25
|
-
|
26
|
-
describe '#kill_by_signal' do
|
27
|
-
[:QUIT, :TERM, :KILL].each do |sig|
|
28
|
-
it "must send #{sig} signal" do
|
29
|
-
pid = rand(1000)
|
30
|
-
expect(Process).to receive(:kill).with(sig, pid)
|
31
|
-
WorkerKiller.kill_by_signal(logger, 11111, sig, pid)
|
32
|
-
end
|
33
|
-
end
|
34
|
-
end
|
35
|
-
|
36
|
-
describe '#kill_by_passenger' do
|
37
|
-
it "must run passenger-config" do
|
38
|
-
pid = rand(1000)
|
39
|
-
path = "passenger-config-#{rand(1000)}"
|
40
|
-
expect(Kernel).to receive(:system).with("#{path} detach-process #{pid}").and_return(true)
|
41
|
-
|
42
|
-
thread = WorkerKiller.kill_by_passenger(logger, 11111, path, pid)
|
43
|
-
thread.join
|
44
|
-
end
|
45
|
-
end
|
46
|
-
|
47
|
-
describe '#kill_self' do
|
48
|
-
context 'with use_quit TRUE' do
|
49
|
-
around do |example|
|
50
|
-
prev = WorkerKiller.configuration
|
51
|
-
config.use_quit = true
|
52
|
-
WorkerKiller.configuration = config
|
53
|
-
example.run
|
54
|
-
ensure
|
55
|
-
WorkerKiller.configuration = prev
|
56
|
-
end
|
57
|
-
|
58
|
-
it 'expect right signal order' do
|
59
|
-
expect(WorkerKiller).to receive(:kill_by_signal).with(logger, anything, :QUIT, anything).exactly(2).times
|
60
|
-
expect(WorkerKiller).to receive(:kill_by_signal).with(logger, anything, :TERM, anything).exactly(2).times
|
61
|
-
expect(WorkerKiller).to receive(:kill_by_signal).with(logger, anything, :KILL, anything).exactly(5).times
|
62
|
-
|
63
|
-
2.times { WorkerKiller.kill_self(logger, Time.now) } # 2 QUIT
|
64
|
-
2.times { WorkerKiller.kill_self(logger, Time.now) } # 2 TERM
|
65
|
-
5.times { WorkerKiller.kill_self(logger, Time.now) } # other - KILL
|
66
|
-
end
|
67
|
-
end
|
68
|
-
|
69
|
-
context 'with use_quit FALSE' do
|
70
|
-
around do |example|
|
71
|
-
prev = WorkerKiller.configuration
|
72
|
-
config.use_quit = false
|
73
|
-
WorkerKiller.configuration = config
|
74
|
-
example.run
|
75
|
-
ensure
|
76
|
-
WorkerKiller.configuration = prev
|
77
|
-
end
|
78
|
-
|
79
|
-
it 'expect right signal order' do
|
80
|
-
expect(WorkerKiller).to receive(:kill_by_signal).with(logger, anything, :TERM, anything).exactly(2).times
|
81
|
-
expect(WorkerKiller).to receive(:kill_by_signal).with(logger, anything, :KILL, anything).exactly(5).times
|
82
|
-
|
83
|
-
2.times { WorkerKiller.kill_self(logger, Time.now) } # 2 TERM
|
84
|
-
5.times { WorkerKiller.kill_self(logger, Time.now) } # other - KILL
|
85
|
-
end
|
86
|
-
end
|
87
|
-
end
|
88
15
|
end
|
89
16
|
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: worker_killer
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.1.1.
|
4
|
+
version: 0.1.1.39838
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Samoilenko Yuri
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2020-
|
11
|
+
date: 2020-10-28 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: get_process_mem
|
@@ -127,13 +127,23 @@ files:
|
|
127
127
|
- lib/worker_killer.rb
|
128
128
|
- lib/worker_killer/configuration.rb
|
129
129
|
- lib/worker_killer/count_limiter.rb
|
130
|
+
- lib/worker_killer/delayed_job_plugin.rb
|
131
|
+
- lib/worker_killer/killer.rb
|
132
|
+
- lib/worker_killer/killer/delayed_job.rb
|
133
|
+
- lib/worker_killer/killer/passenger.rb
|
134
|
+
- lib/worker_killer/killer/signal.rb
|
130
135
|
- lib/worker_killer/memory_limiter.rb
|
131
136
|
- lib/worker_killer/middleware.rb
|
132
137
|
- lib/worker_killer/version.rb
|
133
138
|
- spec/count_limiter_spec.rb
|
139
|
+
- spec/killer/delayed_job_spec.rb
|
140
|
+
- spec/killer/passenger_spec.rb
|
141
|
+
- spec/killer/signal_spec.rb
|
142
|
+
- spec/killer_spec.rb
|
134
143
|
- spec/memory_limiter_spec.rb
|
135
144
|
- spec/middleware_spec.rb
|
136
145
|
- spec/spec_helper.rb
|
146
|
+
- spec/support/logger.rb
|
137
147
|
- spec/worker_killer_spec.rb
|
138
148
|
homepage: https://github.com/RnD-Soft/worker_killer
|
139
149
|
licenses:
|
@@ -160,7 +170,12 @@ specification_version: 4
|
|
160
170
|
summary: Kill any workers by memory and request counts or take custom reaction
|
161
171
|
test_files:
|
162
172
|
- spec/middleware_spec.rb
|
173
|
+
- spec/killer_spec.rb
|
163
174
|
- spec/spec_helper.rb
|
175
|
+
- spec/killer/passenger_spec.rb
|
176
|
+
- spec/killer/signal_spec.rb
|
177
|
+
- spec/killer/delayed_job_spec.rb
|
164
178
|
- spec/memory_limiter_spec.rb
|
179
|
+
- spec/support/logger.rb
|
165
180
|
- spec/worker_killer_spec.rb
|
166
181
|
- spec/count_limiter_spec.rb
|