worker_killer 1.1.0.214159 → 1.2.0.338563
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/worker_killer/killer/puma.rb +19 -9
- data/lib/worker_killer/killer.rb +1 -1
- data/lib/worker_killer/middleware.rb +61 -6
- data/lib/worker_killer/puma/plugin/worker_killer_ng.rb +13 -0
- data/lib/worker_killer/puma_plugin.rb +81 -9
- data/lib/worker_killer/puma_plugin_ng.rb +140 -0
- data/lib/worker_killer/version.rb +1 -1
- data/spec/killer/puma_spec.rb +17 -22
- data/spec/middleware_spec.rb +34 -10
- data/spec/puma_plugin_ng_spec.rb +65 -0
- data/spec/puma_plugin_spec.rb +46 -4
- data/spec/spec_helper.rb +1 -0
- metadata +19 -15
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 64ffd6e8f669eba3536a04cd6910c9193a3a4177640cc8bdfa461c829a97b0e7
|
|
4
|
+
data.tar.gz: 0b4a46d57ec83559630f31d0c8648b809b7edd923c6f6acd45636204387e9070
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: a59dc746b5c4a1ae2d457a590719191e0698231833b6dba31af81ec4e1548b33460118ffc1557ff860263c0ad52b6d699d67f0304f9680e6e82869de3709240a
|
|
7
|
+
data.tar.gz: 0faf8d3cf8347a367cfa988541dbca1727bb4f8d70302f6bde81b74800b23a506a394cb78bf4e77f470fc5e82f80c26beb836314dbdde72bf02213d3a0b4655e
|
|
@@ -1,27 +1,37 @@
|
|
|
1
|
-
require 'socket'
|
|
2
|
-
|
|
3
1
|
module ::WorkerKiller
|
|
4
2
|
module Killer
|
|
5
3
|
class Puma < ::WorkerKiller::Killer::Base
|
|
6
4
|
|
|
7
|
-
attr_accessor :
|
|
5
|
+
attr_accessor :worker_num
|
|
8
6
|
|
|
9
|
-
def initialize(
|
|
7
|
+
def initialize(puma_plugin:, worker_num: nil, **kwargs)
|
|
10
8
|
super(**kwargs)
|
|
11
|
-
@
|
|
9
|
+
@puma_plugin = puma_plugin
|
|
12
10
|
@worker_num = worker_num
|
|
13
11
|
end
|
|
14
12
|
|
|
15
13
|
def do_kill(sig, pid, alive_sec, **_params)
|
|
14
|
+
return if @worker_num.nil? # May be nil if Puma not in Cluster mode
|
|
16
15
|
return if @already_sended == sig
|
|
17
16
|
|
|
18
|
-
logger.warn "#{self} send #{worker_num} to Puma Plugin (pid: #{pid}) alive: #{alive_sec} sec (trial #{kill_attempts})"
|
|
17
|
+
logger.warn "#{self.class} send [W#{worker_num}] to Puma Plugin (from pid: #{pid}) alive: #{alive_sec} sec (trial #{kill_attempts}) triggered by #{sig}"
|
|
19
18
|
|
|
20
19
|
@already_sended = sig
|
|
20
|
+
@puma_plugin.set_logger!(logger)
|
|
21
|
+
|
|
22
|
+
@puma_plugin.request_restart_server(worker_num)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def do_inhibit(_path_info)
|
|
26
|
+
@puma_plugin.set_logger!(logger)
|
|
27
|
+
|
|
28
|
+
@puma_plugin.inhibit_restart(worker_num)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def do_release
|
|
32
|
+
@puma_plugin.set_logger!(logger)
|
|
21
33
|
|
|
22
|
-
|
|
23
|
-
sock.puts Integer(worker_num).to_s
|
|
24
|
-
end
|
|
34
|
+
@puma_plugin.release_restart(worker_num)
|
|
25
35
|
end
|
|
26
36
|
|
|
27
37
|
end
|
data/lib/worker_killer/killer.rb
CHANGED
|
@@ -4,23 +4,78 @@ require 'worker_killer/count_limiter'
|
|
|
4
4
|
module WorkerKiller
|
|
5
5
|
class Middleware
|
|
6
6
|
|
|
7
|
-
attr_reader :limiter, :killer, :reaction
|
|
7
|
+
attr_reader :limiter, :killer, :reaction, :inhibitions
|
|
8
8
|
|
|
9
|
-
|
|
9
|
+
# inhibitions - список адресов, которые будут времнно блокировать перезапуск воркеров:
|
|
10
|
+
# Rails.application.config.middleware.insert_before(
|
|
11
|
+
# Rack::Sendfile,
|
|
12
|
+
# WorkerKiller::Middleware::RequestsLimiter, killer:, min: 2, max: 3, inhibitions: ['/test']
|
|
13
|
+
# )
|
|
14
|
+
#
|
|
15
|
+
def initialize(app, killer:, klass:, inhibitions: [], reaction: nil, **opts)
|
|
10
16
|
@app = app
|
|
11
17
|
@killer = killer
|
|
12
18
|
|
|
13
|
-
@
|
|
14
|
-
|
|
19
|
+
@inhibitions = if @killer.respond_to?(:do_inhibit)
|
|
20
|
+
inhibitions.dup.freeze
|
|
21
|
+
else
|
|
22
|
+
[].freeze
|
|
15
23
|
end
|
|
16
24
|
|
|
25
|
+
@reaction = reaction || method(:default_kill)
|
|
26
|
+
|
|
17
27
|
@limiter = klass.new(**opts)
|
|
18
28
|
end
|
|
19
29
|
|
|
20
30
|
def call(env)
|
|
21
|
-
|
|
31
|
+
inhibited = false
|
|
32
|
+
if (path_info = env['PATH_INFO']) && inhibitions.any?{|i| path_info[i] }
|
|
33
|
+
inihibit(path_info)
|
|
34
|
+
inhibited = true
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
tuple = @app.call(env)
|
|
38
|
+
tuple = with_inhibition(tuple) if inhibited
|
|
39
|
+
|
|
40
|
+
tuple
|
|
22
41
|
ensure
|
|
23
|
-
reaction.call(limiter, killer) if limiter.check
|
|
42
|
+
reaction.call(limiter, killer) if killer && limiter.check
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def with_inhibition(tuple)
|
|
46
|
+
if tuple[2].respond_to?(:each)
|
|
47
|
+
old = tuple[2]
|
|
48
|
+
tuple[2] = Enumerator.new do |y|
|
|
49
|
+
old.each do |chunk|
|
|
50
|
+
y << chunk
|
|
51
|
+
end
|
|
52
|
+
old.close if old.respond_to?(:close)
|
|
53
|
+
release
|
|
54
|
+
end
|
|
55
|
+
else
|
|
56
|
+
release
|
|
57
|
+
end
|
|
58
|
+
tuple
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def default_kill(l, k)
|
|
62
|
+
k.kill(l.started_at)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def inihibit(path_info)
|
|
66
|
+
killer.do_inhibit(path_info)
|
|
67
|
+
rescue StandardError => e
|
|
68
|
+
logger.error "#{self.class}::inhibit error: #{e.inspect}"
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
def release
|
|
72
|
+
killer.do_release
|
|
73
|
+
rescue StandardError => e
|
|
74
|
+
logger.error "#{self.class}::release error: #{e.inspect}"
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def logger
|
|
78
|
+
@logger ||= WorkerKiller.configuration.logger
|
|
24
79
|
end
|
|
25
80
|
|
|
26
81
|
class RequestsLimiter < ::WorkerKiller::Middleware
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
require 'worker_killer'
|
|
2
|
+
require 'worker_killer/puma_plugin_ng'
|
|
3
|
+
|
|
4
|
+
Puma::Plugin.create do
|
|
5
|
+
def config(cfg)
|
|
6
|
+
::WorkerKiller::PumaPluginNg.instance.config(cfg)
|
|
7
|
+
end
|
|
8
|
+
|
|
9
|
+
def start(launcher)
|
|
10
|
+
::WorkerKiller::PumaPluginNg.instance.start(launcher)
|
|
11
|
+
end
|
|
12
|
+
end
|
|
13
|
+
|
|
@@ -1,30 +1,62 @@
|
|
|
1
|
+
require 'delegate'
|
|
1
2
|
require 'singleton'
|
|
3
|
+
require 'socket'
|
|
2
4
|
|
|
3
5
|
require 'worker_killer/memory_limiter'
|
|
4
6
|
require 'worker_killer/count_limiter'
|
|
5
7
|
|
|
6
|
-
|
|
7
8
|
module WorkerKiller
|
|
8
9
|
class PumaPlugin
|
|
9
10
|
|
|
10
11
|
include Singleton
|
|
11
12
|
|
|
13
|
+
class PumaLogWrapper < SimpleDelegator
|
|
14
|
+
|
|
15
|
+
def info(msg)
|
|
16
|
+
__getobj__.log(msg)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def warn(msg)
|
|
20
|
+
__getobj__.log(msg)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def debug(msg)
|
|
24
|
+
__getobj__.log("(DEBUG) #{msg}")
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
end
|
|
28
|
+
|
|
12
29
|
attr_accessor :ipc_path, :killer, :thread
|
|
13
30
|
|
|
14
31
|
def initialize
|
|
32
|
+
@killer = ::WorkerKiller::Killer::Puma.new(worker_num: nil, puma_plugin: self)
|
|
33
|
+
@worker_num = nil
|
|
34
|
+
@debug = false
|
|
35
|
+
|
|
15
36
|
@ipc_path = File.join('tmp', "puma_worker_killer_#{Process.pid}.socket")
|
|
16
|
-
@killer = ::WorkerKiller::Killer::Puma.new(worker_num: nil, ipc_path: ipc_path)
|
|
17
|
-
log "Initializing IPC: #{@ipc_path}"
|
|
18
37
|
end
|
|
19
38
|
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
39
|
+
# Этот метод зовётся при ИНИЦИАЛИЗАЦИИ плагина внути master-процесса, в самомо начале
|
|
40
|
+
# тут можно выполнить конфигурацию чегоднибудь нужного
|
|
41
|
+
def config(dsl)
|
|
42
|
+
if %w[t 1].include?(ENV.fetch('WORKER_KILLER_DEBUG', 'false').to_s.downcase[0].to_s)
|
|
43
|
+
@debug = true
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
dsl.before_worker_boot do |num|
|
|
23
47
|
@killer.worker_num = num
|
|
48
|
+
@worker_num = num
|
|
49
|
+
@tag = nil
|
|
50
|
+
log "Set worker_num: #{num}"
|
|
24
51
|
end
|
|
25
52
|
end
|
|
26
53
|
|
|
54
|
+
# Этот метод зовётся при ИНИЦИАЛИЗАЦИИ плагина внути master-процесса, контролирующего кластер Puma
|
|
55
|
+
# псле форка данные сохранённые тут также доступны (например logger)
|
|
27
56
|
def start(launcher)
|
|
57
|
+
set_logger!(PumaLogWrapper.new(launcher.log_writer))
|
|
58
|
+
|
|
59
|
+
log "Initializing IPC: #{@ipc_path}"
|
|
28
60
|
@runner = launcher.instance_variable_get('@runner')
|
|
29
61
|
|
|
30
62
|
launcher.events.on_booted do
|
|
@@ -32,10 +64,32 @@ module WorkerKiller
|
|
|
32
64
|
end
|
|
33
65
|
end
|
|
34
66
|
|
|
67
|
+
def set_logger!(logger)
|
|
68
|
+
@logger = logger
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Этот метод зовётся из Middleware внтури воркера
|
|
72
|
+
def request_restart_server(worker_num)
|
|
73
|
+
log("Equeue worker #{worker_num} for restarting...")
|
|
74
|
+
Socket.unix(ipc_path) do |sock|
|
|
75
|
+
sock.puts Integer(worker_num).to_s
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Этот метод зовётся из Middleware внтури воркера
|
|
80
|
+
def inhibit_restart(_worker_num)
|
|
81
|
+
nil
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Этот метод зовётся из Middleware внтури воркера
|
|
85
|
+
def release_restart(_worker_num)
|
|
86
|
+
nil
|
|
87
|
+
end
|
|
88
|
+
|
|
35
89
|
def start_ipc_listener
|
|
36
|
-
log
|
|
90
|
+
log "Start IPC listener on #{@ipc_path}"
|
|
37
91
|
Thread.new do
|
|
38
|
-
Socket.unix_server_loop(ipc_path) do |sock, *
|
|
92
|
+
Socket.unix_server_loop(ipc_path) do |sock, *_args|
|
|
39
93
|
if (line = sock.gets)
|
|
40
94
|
worker_num = Integer(line.strip)
|
|
41
95
|
if (worker = find_worker(worker_num))
|
|
@@ -72,7 +126,25 @@ module WorkerKiller
|
|
|
72
126
|
end
|
|
73
127
|
|
|
74
128
|
def log(msg)
|
|
75
|
-
|
|
129
|
+
if @logger
|
|
130
|
+
@logger.warn("#{tag} #{msg}")
|
|
131
|
+
else
|
|
132
|
+
warn("[#{Process.pid}] #{tag} #{msg}")
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def tag
|
|
137
|
+
@tag ||= "[#{self.class}] #{@worker_num.nil? ? '[M]' : "[W#{@worker_num}]"}"
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
def debug(msg)
|
|
141
|
+
return unless @debug
|
|
142
|
+
|
|
143
|
+
if @logger
|
|
144
|
+
@logger.debug("#{tag} #{msg}")
|
|
145
|
+
else
|
|
146
|
+
warn("[#{Process.pid}] #{tag} (DEBUG) #{msg}")
|
|
147
|
+
end
|
|
76
148
|
end
|
|
77
149
|
|
|
78
150
|
end
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
require 'delegate'
|
|
2
|
+
require 'singleton'
|
|
3
|
+
|
|
4
|
+
require 'worker_killer/memory_limiter'
|
|
5
|
+
require 'worker_killer/count_limiter'
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
module WorkerKiller
|
|
9
|
+
class PumaPluginNg
|
|
10
|
+
|
|
11
|
+
include Singleton
|
|
12
|
+
|
|
13
|
+
class PumaLogWrapper < SimpleDelegator
|
|
14
|
+
|
|
15
|
+
def info(msg)
|
|
16
|
+
__getobj__.log(msg)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
def warn(msg)
|
|
20
|
+
__getobj__.log(msg)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def debug(msg)
|
|
24
|
+
__getobj__.log("(DEBUG) #{msg}")
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
attr_accessor :killer, :puma_server, :inhibited, :kill_queue
|
|
30
|
+
|
|
31
|
+
def initialize
|
|
32
|
+
@killer = ::WorkerKiller::Killer::Puma.new(worker_num: nil, puma_plugin: self)
|
|
33
|
+
@worker_num = nil
|
|
34
|
+
@debug = false
|
|
35
|
+
|
|
36
|
+
@puma_server = nil
|
|
37
|
+
|
|
38
|
+
@force_restart = false
|
|
39
|
+
@inhibited ||= Hash.new {|h, k| h[k] = 0 }
|
|
40
|
+
@kill_queue ||= Set.new
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# Этот метод зовётся при ИНИЦИАЛИЗАЦИИ плагина внути master-процесса, в самомо начале
|
|
44
|
+
# тут можно выполнить конфигурацию чегоднибудь нужного
|
|
45
|
+
def config(dsl)
|
|
46
|
+
if %w[t 1].include?(ENV.fetch('WORKER_KILLER_DEBUG', 'false').to_s.downcase[0].to_s)
|
|
47
|
+
@debug = true
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
dsl.before_worker_boot do |num|
|
|
51
|
+
@killer.worker_num = num
|
|
52
|
+
@worker_num = num
|
|
53
|
+
@tag = nil
|
|
54
|
+
log "Set worker_num: #{num}"
|
|
55
|
+
end
|
|
56
|
+
|
|
57
|
+
dsl.out_of_band do
|
|
58
|
+
do_kill('OOB') unless @force_restart
|
|
59
|
+
end
|
|
60
|
+
end
|
|
61
|
+
|
|
62
|
+
# Этот метод зовётся при ИНИЦИАЛИЗАЦИИ плагина внути master-процесса, контролирующего кластер Puma
|
|
63
|
+
# псле форка данные сохранённые тут также доступны (например logger)
|
|
64
|
+
def start(launcher)
|
|
65
|
+
set_logger!(PumaLogWrapper.new(launcher.log_writer))
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def set_logger!(logger)
|
|
69
|
+
@logger = logger
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Завершать процесс сразу после окончания inhibited метода. Иначе завершение будет происзодть в Out Of Band методе
|
|
73
|
+
def force_restart!(force = true)
|
|
74
|
+
@force_restart = force
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Этот метод зовётся из Middleware внтури воркера
|
|
78
|
+
def request_restart_server(worker_num)
|
|
79
|
+
return if @worker_num != worker_num
|
|
80
|
+
|
|
81
|
+
log("Equeue worker #{worker_num} for restarting...")
|
|
82
|
+
kill_queue << worker_num
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
# Этот метод зовётся из Middleware внтури воркера
|
|
86
|
+
def inhibit_restart(worker_num)
|
|
87
|
+
return if @worker_num != worker_num
|
|
88
|
+
|
|
89
|
+
cnt = inhibited[worker_num] += 1 # just increase inhibit counter
|
|
90
|
+
log("Worker inhibition increased: #{cnt}")
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# Этот метод зовётся из Middleware внтури воркера
|
|
94
|
+
def release_restart(worker_num)
|
|
95
|
+
return if @worker_num != worker_num
|
|
96
|
+
|
|
97
|
+
cnt = inhibited[worker_num] -= 1 # just decrease inhibit counter
|
|
98
|
+
log("Worker inhibition decreased: #{cnt}")
|
|
99
|
+
return unless cnt <= 0
|
|
100
|
+
|
|
101
|
+
inhibited.delete(worker_num)
|
|
102
|
+
log('Worker released')
|
|
103
|
+
do_kill('RELEASE') if @force_restart
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def do_kill(name)
|
|
107
|
+
return if kill_queue.empty?
|
|
108
|
+
|
|
109
|
+
log "Killing workers by #{name}: #{kill_queue}"
|
|
110
|
+
(kill_queue - inhibited.keys).each do |worker_num|
|
|
111
|
+
kill_queue.delete(worker_num)
|
|
112
|
+
Thread.current.puma_server.begin_restart
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def log(msg)
|
|
117
|
+
if @logger
|
|
118
|
+
@logger.warn("#{tag} #{msg}")
|
|
119
|
+
else
|
|
120
|
+
warn("[#{Process.pid}] #{tag} #{msg}")
|
|
121
|
+
end
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
def tag
|
|
125
|
+
@tag ||= "[#{self.class}] #{@worker_num.nil? ? '[M]' : "[W#{@worker_num}]"}"
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def debug(msg)
|
|
129
|
+
return unless @debug
|
|
130
|
+
|
|
131
|
+
if @logger
|
|
132
|
+
@logger.debug("#{tag} #{msg}")
|
|
133
|
+
else
|
|
134
|
+
warn("[#{Process.pid}] #{tag} (DEBUG) #{msg}")
|
|
135
|
+
end
|
|
136
|
+
end
|
|
137
|
+
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
|
data/spec/killer/puma_spec.rb
CHANGED
|
@@ -1,34 +1,29 @@
|
|
|
1
1
|
RSpec.describe WorkerKiller::Killer::Puma do
|
|
2
|
-
let(:
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
2
|
+
let(:puma_plugin) { double(set_logger!: nil) }
|
|
3
|
+
let(:killer){ described_class.new(puma_plugin: puma_plugin, worker_num: 99) }
|
|
4
|
+
|
|
5
|
+
describe '#kill' do
|
|
6
|
+
it do
|
|
7
|
+
expect(puma_plugin).to receive(:request_restart_server)
|
|
8
|
+
|
|
9
|
+
killer.kill(Time.now)
|
|
6
10
|
end
|
|
7
11
|
end
|
|
8
12
|
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
13
|
+
describe '#do_inhibit' do
|
|
14
|
+
it do
|
|
15
|
+
expect(puma_plugin).to receive(:inhibit_restart)
|
|
12
16
|
|
|
13
|
-
|
|
14
|
-
around do |example|
|
|
15
|
-
prev = WorkerKiller.configuration
|
|
16
|
-
WorkerKiller.configuration = config
|
|
17
|
-
example.run
|
|
18
|
-
ensure
|
|
19
|
-
WorkerKiller.configuration = prev
|
|
17
|
+
killer.do_inhibit('something')
|
|
20
18
|
end
|
|
19
|
+
end
|
|
21
20
|
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
expect(
|
|
21
|
+
describe '#do_release' do
|
|
22
|
+
it do
|
|
23
|
+
expect(puma_plugin).to receive(:release_restart)
|
|
25
24
|
|
|
26
|
-
killer.kill(Time.now)
|
|
27
|
-
expect(buffer.string.strip).to eq(99.to_s)
|
|
28
25
|
|
|
29
|
-
|
|
30
|
-
2.times { killer.kill(Time.now) } # 1 TERM
|
|
31
|
-
5.times { killer.kill(Time.now) } # 5 KILL
|
|
26
|
+
killer.do_release
|
|
32
27
|
end
|
|
33
28
|
end
|
|
34
29
|
end
|
data/spec/middleware_spec.rb
CHANGED
|
@@ -2,13 +2,17 @@ require 'securerandom'
|
|
|
2
2
|
|
|
3
3
|
RSpec.describe WorkerKiller::Middleware do
|
|
4
4
|
let(:app){ double(call: {}) }
|
|
5
|
-
let(:killer) { double }
|
|
5
|
+
let(:killer) { double(WorkerKiller::Killer::Base, do_inhibit: true) }
|
|
6
6
|
let(:reaction){ double }
|
|
7
7
|
let(:anykey){ SecureRandom.hex(8) }
|
|
8
|
+
let(:inhibitions) { [%r{/attachments}] }
|
|
9
|
+
let(:env) { { 'PATH_INFO' => '/something' } }
|
|
8
10
|
|
|
9
11
|
describe 'Custom class' do
|
|
10
12
|
let(:klass){ double }
|
|
11
|
-
let(:options)
|
|
13
|
+
let(:options) do
|
|
14
|
+
{ killer: killer, klass: klass, reaction: reaction, anykey: anykey, inhibitions: inhibitions }
|
|
15
|
+
end
|
|
12
16
|
subject{ described_class.new(app, **options) }
|
|
13
17
|
|
|
14
18
|
it 'is expected to be initialized' do
|
|
@@ -19,7 +23,7 @@ RSpec.describe WorkerKiller::Middleware do
|
|
|
19
23
|
end
|
|
20
24
|
|
|
21
25
|
describe WorkerKiller::Middleware::RequestsLimiter do
|
|
22
|
-
let(:options){ { killer: killer, min: 3, max: 3 } }
|
|
26
|
+
let(:options){ { killer: killer, min: 3, max: 3, inhibitions: inhibitions } }
|
|
23
27
|
subject{ described_class.new(app, **options) }
|
|
24
28
|
|
|
25
29
|
it 'is expected to be initialized without reaction' do
|
|
@@ -30,7 +34,25 @@ RSpec.describe WorkerKiller::Middleware do
|
|
|
30
34
|
expect(killer).to receive(:kill).with(Time).twice
|
|
31
35
|
|
|
32
36
|
4.times do
|
|
33
|
-
subject.call(
|
|
37
|
+
subject.call(env)
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
describe 'inhibitions' do
|
|
43
|
+
let(:env) { { 'PATH_INFO' => '/attachments' } }
|
|
44
|
+
|
|
45
|
+
it 'is expected to use inhibitions' do
|
|
46
|
+
expect(WorkerKiller::CountLimiter).to receive(:new).with(min: 3, max: 3).and_call_original
|
|
47
|
+
expect(subject.limiter).to be_an(WorkerKiller::CountLimiter)
|
|
48
|
+
expect(subject.limiter.min).to eq(3)
|
|
49
|
+
expect(subject.limiter.max).to eq(3)
|
|
50
|
+
expect(killer).to receive(:kill).with(Time).twice
|
|
51
|
+
expect(killer).to receive(:do_release).exactly(4).times
|
|
52
|
+
|
|
53
|
+
4.times do
|
|
54
|
+
subject.call(env)
|
|
55
|
+
end
|
|
34
56
|
end
|
|
35
57
|
end
|
|
36
58
|
|
|
@@ -44,36 +66,38 @@ RSpec.describe WorkerKiller::Middleware do
|
|
|
44
66
|
expect(reaction).to receive(:call).with(subject.limiter, killer).twice
|
|
45
67
|
|
|
46
68
|
4.times do
|
|
47
|
-
subject.call(
|
|
69
|
+
subject.call(env)
|
|
48
70
|
end
|
|
49
71
|
end
|
|
50
72
|
end
|
|
51
73
|
|
|
52
74
|
describe WorkerKiller::Middleware::OOMLimiter do
|
|
53
|
-
let(:options){ { killer: killer, min: 2222, max: 2223 } }
|
|
75
|
+
let(:options){ { killer: killer, min: 2222, max: 2223, inhibitions: inhibitions } }
|
|
54
76
|
subject{ described_class.new(app, **options) }
|
|
55
77
|
|
|
56
78
|
it 'is expected to be initialized without reaction' do
|
|
57
|
-
expect(WorkerKiller::MemoryLimiter).to receive(:new).with(min: 2222,
|
|
79
|
+
expect(WorkerKiller::MemoryLimiter).to receive(:new).with(min: 2222,
|
|
80
|
+
max: 2223).and_call_original
|
|
58
81
|
expect(subject.limiter).to be_an(WorkerKiller::MemoryLimiter)
|
|
59
82
|
expect(subject.limiter.min).to eq(2222)
|
|
60
83
|
expect(subject.limiter.max).to eq(2223)
|
|
61
84
|
expect(killer).to receive(:kill).with(subject.limiter.started_at).once
|
|
62
85
|
expect(subject.limiter).to receive(:check).and_return(true)
|
|
63
86
|
|
|
64
|
-
subject.call(
|
|
87
|
+
subject.call(env)
|
|
65
88
|
end
|
|
66
89
|
|
|
67
90
|
it 'is expected to be initialized with reaction' do
|
|
68
91
|
options[:reaction] = reaction
|
|
69
|
-
expect(WorkerKiller::MemoryLimiter).to receive(:new).with(min: 2222,
|
|
92
|
+
expect(WorkerKiller::MemoryLimiter).to receive(:new).with(min: 2222,
|
|
93
|
+
max: 2223).and_call_original
|
|
70
94
|
expect(subject.limiter).to be_an(WorkerKiller::MemoryLimiter)
|
|
71
95
|
expect(subject.limiter.min).to eq(2222)
|
|
72
96
|
expect(subject.limiter.max).to eq(2223)
|
|
73
97
|
expect(reaction).to receive(:call).with(subject.limiter, killer)
|
|
74
98
|
expect(subject.limiter).to receive(:check).and_return(true)
|
|
75
99
|
|
|
76
|
-
subject.call(
|
|
100
|
+
subject.call(env)
|
|
77
101
|
end
|
|
78
102
|
end
|
|
79
103
|
end
|
|
@@ -0,0 +1,65 @@
|
|
|
1
|
+
RSpec.describe WorkerKiller::PumaPluginNg do
|
|
2
|
+
subject(:plugin){ described_class.send(:new) }
|
|
3
|
+
let(:dsl) { double }
|
|
4
|
+
let(:runner){ double }
|
|
5
|
+
let(:events) { double }
|
|
6
|
+
let(:launcher){ double('@runner' => runner, 'events' => events, log_writer: double(log: nil)) }
|
|
7
|
+
let(:worker){ double('booted?' => true, 'term?' => false) }
|
|
8
|
+
|
|
9
|
+
it {
|
|
10
|
+
is_expected.to have_attributes(killer: ::WorkerKiller::Killer::Puma,
|
|
11
|
+
inhibited: Hash,
|
|
12
|
+
kill_queue: Set)
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
context 'Puma initialization' do
|
|
16
|
+
it 'expected to register CB' do
|
|
17
|
+
expect(dsl).to receive(:before_worker_boot).and_yield(123)
|
|
18
|
+
expect(dsl).to receive(:out_of_band).and_yield
|
|
19
|
+
plugin.config(dsl)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
it do
|
|
23
|
+
expect(plugin).to receive(:set_logger!).and_call_original
|
|
24
|
+
plugin.start(launcher)
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
context '#request_restart_server' do
|
|
29
|
+
it do
|
|
30
|
+
plugin.instance_variable_set('@worker_num', 99)
|
|
31
|
+
expect { plugin.request_restart_server(99) }.to change { plugin.kill_queue }.to([99])
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
context '#inhibit_restart' do
|
|
36
|
+
it do
|
|
37
|
+
plugin.instance_variable_set('@worker_num', 99)
|
|
38
|
+
expect { plugin.inhibit_restart(99) }.to change { plugin.inhibited }.from({}).to({ 99 => 1 })
|
|
39
|
+
end
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
context '#release_restart' do
|
|
43
|
+
it do
|
|
44
|
+
plugin.instance_variable_set('@worker_num', 99)
|
|
45
|
+
plugin.inhibit_restart(99)
|
|
46
|
+
|
|
47
|
+
expect { plugin.release_restart(99) }.to change { plugin.inhibited }.from({ 99 => 1 }).to({})
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
context '#do_kill' do
|
|
52
|
+
it do
|
|
53
|
+
Thread.attr_accessor(:puma_server)
|
|
54
|
+
|
|
55
|
+
plugin.instance_variable_set('@worker_num', 99)
|
|
56
|
+
plugin.request_restart_server(99)
|
|
57
|
+
puma_server = double()
|
|
58
|
+
Thread.current.puma_server = puma_server
|
|
59
|
+
expect(puma_server).to receive(:begin_restart)
|
|
60
|
+
|
|
61
|
+
plugin.do_kill('test')
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
data/spec/puma_plugin_spec.rb
CHANGED
|
@@ -1,9 +1,9 @@
|
|
|
1
1
|
RSpec.describe WorkerKiller::PumaPlugin do
|
|
2
2
|
subject(:plugin){ described_class.instance }
|
|
3
|
-
let(:
|
|
3
|
+
let(:dsl) { double }
|
|
4
4
|
let(:runner){ double }
|
|
5
5
|
let(:events) { double }
|
|
6
|
-
let(:launcher){ double('@runner' => runner, 'events' => events) }
|
|
6
|
+
let(:launcher){ double('@runner' => runner, 'events' => events, log_writer: double(log: nil)) }
|
|
7
7
|
let(:worker){ double('booted?' => true, 'term?' => false) }
|
|
8
8
|
|
|
9
9
|
it {
|
|
@@ -11,10 +11,17 @@ RSpec.describe WorkerKiller::PumaPlugin do
|
|
|
11
11
|
killer: ::WorkerKiller::Killer::Puma)
|
|
12
12
|
}
|
|
13
13
|
|
|
14
|
+
|
|
14
15
|
context 'Puma initialization' do
|
|
16
|
+
it 'expected to register CB' do
|
|
17
|
+
expect(dsl).to receive(:before_worker_boot).and_yield(123)
|
|
18
|
+
plugin.config(dsl)
|
|
19
|
+
end
|
|
20
|
+
|
|
15
21
|
it do
|
|
16
|
-
expect(
|
|
17
|
-
|
|
22
|
+
expect(plugin).to receive(:set_logger!).and_call_original
|
|
23
|
+
expect(events).to receive(:on_booted)
|
|
24
|
+
plugin.start(launcher)
|
|
18
25
|
end
|
|
19
26
|
|
|
20
27
|
it do
|
|
@@ -30,5 +37,40 @@ RSpec.describe WorkerKiller::PumaPlugin do
|
|
|
30
37
|
plugin.thread.join
|
|
31
38
|
end
|
|
32
39
|
end
|
|
40
|
+
|
|
41
|
+
context 'expect right signal order' do
|
|
42
|
+
let(:config) do
|
|
43
|
+
WorkerKiller::Configuration.new.tap do |c|
|
|
44
|
+
c.quit_attempts = 2
|
|
45
|
+
c.term_attempts = 2
|
|
46
|
+
end
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
let(:killer){ ::WorkerKiller::Killer::Puma.new(puma_plugin: plugin, worker_num: 99) }
|
|
50
|
+
let(:buffer) { StringIO.new }
|
|
51
|
+
|
|
52
|
+
around do |example|
|
|
53
|
+
prev = WorkerKiller.configuration
|
|
54
|
+
WorkerKiller.configuration = config
|
|
55
|
+
example.run
|
|
56
|
+
ensure
|
|
57
|
+
WorkerKiller.configuration = prev
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
it 'expect right signal order' do
|
|
62
|
+
expect(Socket).to receive(:unix).with(/puma_worker_.*socket/).and_yield(buffer).exactly(3).times
|
|
63
|
+
expect(Process).not_to receive(:kill)
|
|
64
|
+
|
|
65
|
+
killer.kill(Time.now)
|
|
66
|
+
expect(buffer.string.strip).to eq(99.to_s)
|
|
67
|
+
|
|
68
|
+
1.times { killer.kill(Time.now) } # 1 QUIT
|
|
69
|
+
2.times { killer.kill(Time.now) } # 1 TERM
|
|
70
|
+
5.times { killer.kill(Time.now) } # 5 KILL
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
|
|
33
75
|
end
|
|
34
76
|
|
data/spec/spec_helper.rb
CHANGED
|
@@ -31,6 +31,7 @@ SimpleCov.start
|
|
|
31
31
|
require 'worker_killer'
|
|
32
32
|
require 'worker_killer/delayed_job_plugin'
|
|
33
33
|
require 'worker_killer/puma_plugin'
|
|
34
|
+
require 'worker_killer/puma_plugin_ng'
|
|
34
35
|
|
|
35
36
|
$root = File.join(File.dirname(__dir__), 'spec')
|
|
36
37
|
Dir[File.join(__dir__, 'support', '**', '*.rb')].sort.each {|f| require f }
|
metadata
CHANGED
|
@@ -1,14 +1,14 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: worker_killer
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 1.
|
|
4
|
+
version: 1.2.0.338563
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Samoilenko Yuri
|
|
8
|
-
autorequire:
|
|
8
|
+
autorequire:
|
|
9
9
|
bindir: bin
|
|
10
10
|
cert_chain: []
|
|
11
|
-
date:
|
|
11
|
+
date: 2025-11-13 00:00:00.000000000 Z
|
|
12
12
|
dependencies:
|
|
13
13
|
- !ruby/object:Gem::Dependency
|
|
14
14
|
name: get_process_mem
|
|
@@ -136,7 +136,9 @@ files:
|
|
|
136
136
|
- lib/worker_killer/memory_limiter.rb
|
|
137
137
|
- lib/worker_killer/middleware.rb
|
|
138
138
|
- lib/worker_killer/puma/plugin/worker_killer.rb
|
|
139
|
+
- lib/worker_killer/puma/plugin/worker_killer_ng.rb
|
|
139
140
|
- lib/worker_killer/puma_plugin.rb
|
|
141
|
+
- lib/worker_killer/puma_plugin_ng.rb
|
|
140
142
|
- lib/worker_killer/version.rb
|
|
141
143
|
- spec/count_limiter_spec.rb
|
|
142
144
|
- spec/delayed_job_plugin/jobs_limiter_spec.rb
|
|
@@ -148,6 +150,7 @@ files:
|
|
|
148
150
|
- spec/killer_spec.rb
|
|
149
151
|
- spec/memory_limiter_spec.rb
|
|
150
152
|
- spec/middleware_spec.rb
|
|
153
|
+
- spec/puma_plugin_ng_spec.rb
|
|
151
154
|
- spec/puma_plugin_spec.rb
|
|
152
155
|
- spec/spec_helper.rb
|
|
153
156
|
- spec/support/logger.rb
|
|
@@ -156,7 +159,7 @@ homepage: https://github.com/RnD-Soft/worker_killer
|
|
|
156
159
|
licenses:
|
|
157
160
|
- MIT
|
|
158
161
|
metadata: {}
|
|
159
|
-
post_install_message:
|
|
162
|
+
post_install_message:
|
|
160
163
|
rdoc_options: []
|
|
161
164
|
require_paths:
|
|
162
165
|
- lib
|
|
@@ -171,22 +174,23 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
|
171
174
|
- !ruby/object:Gem::Version
|
|
172
175
|
version: '0'
|
|
173
176
|
requirements: []
|
|
174
|
-
rubygems_version: 3.
|
|
175
|
-
signing_key:
|
|
177
|
+
rubygems_version: 3.2.33
|
|
178
|
+
signing_key:
|
|
176
179
|
specification_version: 4
|
|
177
180
|
summary: Kill any workers by memory and request counts or take custom reaction
|
|
178
181
|
test_files:
|
|
179
|
-
- spec/
|
|
180
|
-
- spec/
|
|
182
|
+
- spec/count_limiter_spec.rb
|
|
183
|
+
- spec/delayed_job_plugin/jobs_limiter_spec.rb
|
|
184
|
+
- spec/delayed_job_plugin/oom_limiter_spec.rb
|
|
181
185
|
- spec/killer/delayed_job_spec.rb
|
|
182
186
|
- spec/killer/passenger_spec.rb
|
|
183
|
-
- spec/
|
|
187
|
+
- spec/killer/puma_spec.rb
|
|
188
|
+
- spec/killer/signal_spec.rb
|
|
184
189
|
- spec/killer_spec.rb
|
|
185
|
-
- spec/
|
|
186
|
-
- spec/worker_killer_spec.rb
|
|
187
|
-
- spec/support/logger.rb
|
|
190
|
+
- spec/memory_limiter_spec.rb
|
|
188
191
|
- spec/middleware_spec.rb
|
|
192
|
+
- spec/puma_plugin_ng_spec.rb
|
|
193
|
+
- spec/puma_plugin_spec.rb
|
|
189
194
|
- spec/spec_helper.rb
|
|
190
|
-
- spec/
|
|
191
|
-
- spec/
|
|
192
|
-
- spec/memory_limiter_spec.rb
|
|
195
|
+
- spec/support/logger.rb
|
|
196
|
+
- spec/worker_killer_spec.rb
|