dylanvaughn-bluepill 0.0.39
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +5 -0
- data/DESIGN.md +10 -0
- data/LICENSE +22 -0
- data/README.md +228 -0
- data/Rakefile +54 -0
- data/VERSION +1 -0
- data/bin/bluepill +103 -0
- data/bin/bpsv +3 -0
- data/bluepill.gemspec +84 -0
- data/lib/bluepill.rb +32 -0
- data/lib/bluepill/application.rb +200 -0
- data/lib/bluepill/application/client.rb +7 -0
- data/lib/bluepill/application/server.rb +24 -0
- data/lib/bluepill/condition_watch.rb +55 -0
- data/lib/bluepill/controller.rb +119 -0
- data/lib/bluepill/dsl.rb +150 -0
- data/lib/bluepill/group.rb +71 -0
- data/lib/bluepill/logger.rb +62 -0
- data/lib/bluepill/process.rb +419 -0
- data/lib/bluepill/process_conditions.rb +13 -0
- data/lib/bluepill/process_conditions/always_true.rb +17 -0
- data/lib/bluepill/process_conditions/cpu_usage.rb +18 -0
- data/lib/bluepill/process_conditions/http.rb +52 -0
- data/lib/bluepill/process_conditions/mem_usage.rb +31 -0
- data/lib/bluepill/process_conditions/process_condition.rb +21 -0
- data/lib/bluepill/process_statistics.rb +24 -0
- data/lib/bluepill/socket.rb +47 -0
- data/lib/bluepill/system.rb +227 -0
- data/lib/bluepill/trigger.rb +60 -0
- data/lib/bluepill/triggers/flapping.rb +59 -0
- data/lib/bluepill/util/rotational_array.rb +66 -0
- data/lib/bluepill/version.rb +3 -0
- data/lib/example.rb +81 -0
- data/lib/runit_example.rb +25 -0
- metadata +167 -0
@@ -0,0 +1,13 @@
|
|
1
|
+
module Bluepill
|
2
|
+
module ProcessConditions
|
3
|
+
def self.[](name)
|
4
|
+
"#{self}::#{name.to_s.camelcase}".constantize
|
5
|
+
end
|
6
|
+
end
|
7
|
+
end
|
8
|
+
|
9
|
+
require "bluepill/process_conditions/process_condition"
|
10
|
+
Dir["#{File.dirname(__FILE__)}/process_conditions/*.rb"].each do |pc|
|
11
|
+
require pc
|
12
|
+
end
|
13
|
+
|
@@ -0,0 +1,18 @@
|
|
1
|
+
module Bluepill
|
2
|
+
module ProcessConditions
|
3
|
+
class CpuUsage < ProcessCondition
|
4
|
+
def initialize(options = {})
|
5
|
+
@below = options[:below]
|
6
|
+
end
|
7
|
+
|
8
|
+
def run(pid)
|
9
|
+
# third col in the ps axu output
|
10
|
+
System.cpu_usage(pid).to_f
|
11
|
+
end
|
12
|
+
|
13
|
+
def check(value)
|
14
|
+
value < @below
|
15
|
+
end
|
16
|
+
end
|
17
|
+
end
|
18
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
require 'uri'
|
3
|
+
|
4
|
+
module Bluepill
|
5
|
+
module ProcessConditions
|
6
|
+
class Http < ProcessCondition
|
7
|
+
def initialize(options = {})
|
8
|
+
@uri = URI.parse(options[:url])
|
9
|
+
@kind = case options[:kind]
|
10
|
+
when Fixnum then Net::HTTPResponse::CODE_TO_OBJ[options[:kind].to_s]
|
11
|
+
when String, Symbol then "Net::HTTP#{options[:kind].to_s.camelize}".constantize
|
12
|
+
else
|
13
|
+
Net::HTTPSuccess
|
14
|
+
end
|
15
|
+
@pattern = options[:pattern] || nil
|
16
|
+
@open_timeout = (options[:open_timeout] || options[:timeout] || 5).to_i
|
17
|
+
@read_timeout = (options[:read_timeout] || options[:timeout] || 5).to_i
|
18
|
+
end
|
19
|
+
|
20
|
+
def run(pid)
|
21
|
+
session = Net::HTTP.new(@uri.host, @uri.port)
|
22
|
+
session.open_timeout = @open_timeout
|
23
|
+
session.read_timeout = @read_timeout
|
24
|
+
hide_net_http_bug do
|
25
|
+
session.start do |http|
|
26
|
+
http.get(@uri.path)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
rescue
|
30
|
+
$!
|
31
|
+
end
|
32
|
+
|
33
|
+
def check(value)
|
34
|
+
return false unless value.kind_of?(@kind)
|
35
|
+
return true unless @pattern
|
36
|
+
return false unless value.class.body_permitted?
|
37
|
+
@pattern === value.body
|
38
|
+
end
|
39
|
+
|
40
|
+
private
|
41
|
+
def hide_net_http_bug
|
42
|
+
yield
|
43
|
+
rescue NoMethodError => e
|
44
|
+
if e.to_s =~ /#{Regexp.escape(%q|undefined method `closed?' for nil:NilClass|)}/
|
45
|
+
raise Errno::ECONNREFUSED, "Connection refused attempting to contact #{@uri.scheme}://#{@uri.host}:#{@uri.port}"
|
46
|
+
else
|
47
|
+
raise
|
48
|
+
end
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,31 @@
|
|
1
|
+
module Bluepill
|
2
|
+
module ProcessConditions
|
3
|
+
class MemUsage < ProcessCondition
|
4
|
+
MB = 1024 ** 2
|
5
|
+
FORMAT_STR = "%d%s"
|
6
|
+
MB_LABEL = "MB"
|
7
|
+
KB_LABEL = "KB"
|
8
|
+
|
9
|
+
def initialize(options = {})
|
10
|
+
@below = options[:below]
|
11
|
+
end
|
12
|
+
|
13
|
+
def run(pid)
|
14
|
+
# rss is on the 5th col
|
15
|
+
System.memory_usage(pid).to_f
|
16
|
+
end
|
17
|
+
|
18
|
+
def check(value)
|
19
|
+
value.kilobytes < @below
|
20
|
+
end
|
21
|
+
|
22
|
+
def format_value(value)
|
23
|
+
if value.kilobytes >= MB
|
24
|
+
FORMAT_STR % [(value / 1024).round, MB_LABEL]
|
25
|
+
else
|
26
|
+
FORMAT_STR % [value, KB_LABEL]
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
@@ -0,0 +1,21 @@
|
|
1
|
+
module Bluepill
|
2
|
+
module ProcessConditions
|
3
|
+
class ProcessCondition
|
4
|
+
def initialize(options = {})
|
5
|
+
@options = options
|
6
|
+
end
|
7
|
+
|
8
|
+
def run(pid)
|
9
|
+
raise "Implement in subclass!"
|
10
|
+
end
|
11
|
+
|
12
|
+
def check(value)
|
13
|
+
raise "Implement in subclass!"
|
14
|
+
end
|
15
|
+
|
16
|
+
def format_value(value)
|
17
|
+
value
|
18
|
+
end
|
19
|
+
end
|
20
|
+
end
|
21
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
module Bluepill
|
2
|
+
class ProcessStatistics
|
3
|
+
STRFTIME = "%m/%d/%Y %H:%I:%S"
|
4
|
+
# possibly persist this data.
|
5
|
+
def initialize
|
6
|
+
@events = Util::RotationalArray.new(10)
|
7
|
+
end
|
8
|
+
|
9
|
+
def record_event(event, reason)
|
10
|
+
@events.push([event, reason, Time.now])
|
11
|
+
end
|
12
|
+
|
13
|
+
def to_s
|
14
|
+
str = []
|
15
|
+
@events.each do |(event, reason, time)|
|
16
|
+
str << " #{event} at #{time.strftime(STRFTIME)} - #{reason || "unspecified"}"
|
17
|
+
end
|
18
|
+
if str.size > 0
|
19
|
+
str << "event history:"
|
20
|
+
end
|
21
|
+
str.reverse.join("\n")
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,47 @@
|
|
1
|
+
require 'socket'
|
2
|
+
|
3
|
+
module Bluepill
|
4
|
+
module Socket
|
5
|
+
TIMEOUT = 10
|
6
|
+
|
7
|
+
extend self
|
8
|
+
|
9
|
+
def client(base_dir, name, &b)
|
10
|
+
UNIXSocket.open(socket_path(base_dir, name), &b)
|
11
|
+
end
|
12
|
+
|
13
|
+
def client_command(base_dir, name, command)
|
14
|
+
client(base_dir, name) do |socket|
|
15
|
+
Timeout.timeout(TIMEOUT) do
|
16
|
+
socket.puts command
|
17
|
+
Marshal.load(socket)
|
18
|
+
end
|
19
|
+
end
|
20
|
+
rescue EOFError, Timeout::Error
|
21
|
+
abort("Socket Timeout: Server may not be responding")
|
22
|
+
end
|
23
|
+
|
24
|
+
def server(base_dir, name)
|
25
|
+
socket_path = self.socket_path(base_dir, name)
|
26
|
+
begin
|
27
|
+
UNIXServer.open(socket_path)
|
28
|
+
rescue Errno::EADDRINUSE
|
29
|
+
# if sock file has been created. test to see if there is a server
|
30
|
+
begin
|
31
|
+
UNIXSocket.open(socket_path)
|
32
|
+
rescue Errno::ECONNREFUSED
|
33
|
+
File.delete(socket_path)
|
34
|
+
return UNIXServer.open(socket_path)
|
35
|
+
else
|
36
|
+
logger.err("Server is already running!")
|
37
|
+
exit(7)
|
38
|
+
end
|
39
|
+
end
|
40
|
+
end
|
41
|
+
|
42
|
+
def socket_path(base_dir, name)
|
43
|
+
File.join(base_dir, 'socks', name + ".sock")
|
44
|
+
end
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
@@ -0,0 +1,227 @@
|
|
1
|
+
require 'etc'
|
2
|
+
|
3
|
+
module Bluepill
|
4
|
+
# This class represents the system that bluepill is running on.. It's mainly used to memoize
|
5
|
+
# results of running ps auxx etc so that every watch in the every process will not result in a fork
|
6
|
+
module System
|
7
|
+
APPEND_MODE = "a"
|
8
|
+
extend self
|
9
|
+
|
10
|
+
# The position of each field in ps output
|
11
|
+
IDX_MAP = {
|
12
|
+
:pid => 0,
|
13
|
+
:ppid => 1,
|
14
|
+
:pcpu => 2,
|
15
|
+
:rss => 3
|
16
|
+
}
|
17
|
+
|
18
|
+
def pid_alive?(pid)
|
19
|
+
begin
|
20
|
+
::Process.kill(0, pid)
|
21
|
+
true
|
22
|
+
rescue Errno::ESRCH
|
23
|
+
false
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def cpu_usage(pid)
|
28
|
+
ps_axu[pid] && ps_axu[pid][IDX_MAP[:pcpu]].to_f
|
29
|
+
end
|
30
|
+
|
31
|
+
def memory_usage(pid)
|
32
|
+
ps_axu[pid] && ps_axu[pid][IDX_MAP[:rss]].to_f
|
33
|
+
end
|
34
|
+
|
35
|
+
def get_children(parent_pid)
|
36
|
+
returning(Array.new) do |child_pids|
|
37
|
+
ps_axu.each_pair do |pid, chunks|
|
38
|
+
child_pids << chunks[IDX_MAP[:pid]].to_i if chunks[IDX_MAP[:ppid]].to_i == parent_pid.to_i
|
39
|
+
end
|
40
|
+
end
|
41
|
+
end
|
42
|
+
|
43
|
+
# Returns the pid of the child that executes the cmd
|
44
|
+
def daemonize(cmd, options = {})
|
45
|
+
rd, wr = IO.pipe
|
46
|
+
|
47
|
+
if child = Daemonize.safefork
|
48
|
+
# we do not wanna create zombies, so detach ourselves from the child exit status
|
49
|
+
::Process.detach(child)
|
50
|
+
|
51
|
+
# parent
|
52
|
+
wr.close
|
53
|
+
|
54
|
+
daemon_id = rd.read.to_i
|
55
|
+
rd.close
|
56
|
+
|
57
|
+
return daemon_id if daemon_id > 0
|
58
|
+
|
59
|
+
else
|
60
|
+
# child
|
61
|
+
rd.close
|
62
|
+
|
63
|
+
drop_privileges(options[:uid], options[:gid])
|
64
|
+
|
65
|
+
# if we cannot write the pid file as the provided user, err out
|
66
|
+
exit unless can_write_pid_file(options[:pid_file], options[:logger])
|
67
|
+
|
68
|
+
to_daemonize = lambda do
|
69
|
+
# Setting end PWD env emulates bash behavior when dealing with symlinks
|
70
|
+
Dir.chdir(ENV["PWD"] = options[:working_dir]) if options[:working_dir]
|
71
|
+
options[:environment].each { |key, value| ENV[key] = value }
|
72
|
+
|
73
|
+
redirect_io(*options.values_at(:stdin, :stdout, :stderr))
|
74
|
+
|
75
|
+
::Kernel.exec(cmd)
|
76
|
+
exit
|
77
|
+
end
|
78
|
+
|
79
|
+
daemon_id = Daemonize.call_as_daemon(to_daemonize, nil, cmd)
|
80
|
+
|
81
|
+
File.open(options[:pid_file], "w") {|f| f.write(daemon_id)}
|
82
|
+
|
83
|
+
wr.write daemon_id
|
84
|
+
wr.close
|
85
|
+
|
86
|
+
exit
|
87
|
+
end
|
88
|
+
end
|
89
|
+
|
90
|
+
# Returns the stdout, stderr and exit code of the cmd
|
91
|
+
def execute_blocking(cmd, options = {})
|
92
|
+
rd, wr = IO.pipe
|
93
|
+
|
94
|
+
if child = Daemonize.safefork
|
95
|
+
# parent
|
96
|
+
wr.close
|
97
|
+
|
98
|
+
cmd_status = rd.read
|
99
|
+
rd.close
|
100
|
+
|
101
|
+
::Process.waitpid(child)
|
102
|
+
|
103
|
+
return Marshal.load(cmd_status)
|
104
|
+
|
105
|
+
else
|
106
|
+
# child
|
107
|
+
rd.close
|
108
|
+
|
109
|
+
# create a child in which we can override the stdin, stdout and stderr
|
110
|
+
cmd_out_read, cmd_out_write = IO.pipe
|
111
|
+
cmd_err_read, cmd_err_write = IO.pipe
|
112
|
+
|
113
|
+
pid = fork {
|
114
|
+
# grandchild
|
115
|
+
drop_privileges(options[:uid], options[:gid])
|
116
|
+
|
117
|
+
Dir.chdir(ENV["PWD"] = options[:working_dir]) if options[:working_dir]
|
118
|
+
options[:environment].each { |key, value| ENV[key] = value }
|
119
|
+
|
120
|
+
# close unused fds so ancestors wont hang. This line is the only reason we are not
|
121
|
+
# using something like popen3. If this fd is not closed, the .read call on the parent
|
122
|
+
# will never return because "wr" would still be open in the "exec"-ed cmd
|
123
|
+
wr.close
|
124
|
+
|
125
|
+
# we do not care about stdin of cmd
|
126
|
+
STDIN.reopen("/dev/null")
|
127
|
+
|
128
|
+
# point stdout of cmd to somewhere we can read
|
129
|
+
cmd_out_read.close
|
130
|
+
STDOUT.reopen(cmd_out_write)
|
131
|
+
cmd_out_write.close
|
132
|
+
|
133
|
+
# same thing for stderr
|
134
|
+
cmd_err_read.close
|
135
|
+
STDERR.reopen(cmd_err_write)
|
136
|
+
cmd_err_write.close
|
137
|
+
|
138
|
+
# finally, replace grandchild with cmd
|
139
|
+
::Kernel.exec(cmd)
|
140
|
+
}
|
141
|
+
|
142
|
+
# we do not use these ends of the pipes in the child
|
143
|
+
cmd_out_write.close
|
144
|
+
cmd_err_write.close
|
145
|
+
|
146
|
+
# wait for the cmd to finish executing and acknowledge it's death
|
147
|
+
::Process.waitpid(pid)
|
148
|
+
|
149
|
+
# collect stdout, stderr and exitcode
|
150
|
+
result = {
|
151
|
+
:stdout => cmd_out_read.read,
|
152
|
+
:stderr => cmd_err_read.read,
|
153
|
+
:exit_code => $?.exitstatus
|
154
|
+
}
|
155
|
+
|
156
|
+
# We're done with these ends of the pipes as well
|
157
|
+
cmd_out_read.close
|
158
|
+
cmd_err_read.close
|
159
|
+
|
160
|
+
# Time to tell the parent about what went down
|
161
|
+
wr.write Marshal.dump(result)
|
162
|
+
wr.close
|
163
|
+
|
164
|
+
exit
|
165
|
+
end
|
166
|
+
end
|
167
|
+
|
168
|
+
def store
|
169
|
+
@store ||= Hash.new
|
170
|
+
end
|
171
|
+
|
172
|
+
def reset_data
|
173
|
+
store.clear unless store.empty?
|
174
|
+
end
|
175
|
+
|
176
|
+
def ps_axu
|
177
|
+
# TODO: need a mutex here
|
178
|
+
store[:ps_axu] ||= begin
|
179
|
+
# BSD style ps invocation
|
180
|
+
lines = `ps axo pid=,ppid=,pcpu=,rss=`.split("\n")
|
181
|
+
|
182
|
+
lines.inject(Hash.new) do |mem, line|
|
183
|
+
chunks = line.split(/\s+/)
|
184
|
+
chunks.delete_if {|c| c.strip.empty? }
|
185
|
+
pid = chunks[IDX_MAP[:pid]].strip.to_i
|
186
|
+
mem[pid] = chunks
|
187
|
+
mem
|
188
|
+
end
|
189
|
+
end
|
190
|
+
end
|
191
|
+
|
192
|
+
# be sure to call this from a fork otherwise it will modify the attributes
|
193
|
+
# of the bluepill daemon
|
194
|
+
def drop_privileges(uid, gid)
|
195
|
+
uid_num = Etc.getpwnam(uid).uid if uid
|
196
|
+
gid_num = Etc.getgrnam(gid).gid if gid
|
197
|
+
|
198
|
+
::Process.groups = [gid_num] if gid
|
199
|
+
::Process::Sys.setgid(gid_num) if gid
|
200
|
+
::Process::Sys.setuid(uid_num) if uid
|
201
|
+
end
|
202
|
+
|
203
|
+
def can_write_pid_file(pid_file, logger)
|
204
|
+
FileUtils.touch(pid_file)
|
205
|
+
File.unlink(pid_file)
|
206
|
+
return true
|
207
|
+
|
208
|
+
rescue Exception => e
|
209
|
+
logger.warning "%s - %s" % [e.class.name, e.message]
|
210
|
+
e.backtrace.each {|l| logger.warning l}
|
211
|
+
return false
|
212
|
+
end
|
213
|
+
|
214
|
+
def redirect_io(io_in, io_out, io_err)
|
215
|
+
$stdin.reopen(io_in) if io_in
|
216
|
+
|
217
|
+
if !io_out.nil? && !io_err.nil? && io_out == io_err
|
218
|
+
$stdout.reopen(io_out, APPEND_MODE)
|
219
|
+
$stderr.reopen($stdout)
|
220
|
+
|
221
|
+
else
|
222
|
+
$stdout.reopen(io_out, APPEND_MODE) if io_out
|
223
|
+
$stderr.reopen(io_err, APPEND_MODE) if io_err
|
224
|
+
end
|
225
|
+
end
|
226
|
+
end
|
227
|
+
end
|