cloud66-bluepill 0.0.62
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/.gitignore +10 -0
- data/.rspec +1 -0
- data/DESIGN.md +10 -0
- data/Gemfile +10 -0
- data/LICENSE +22 -0
- data/README.md +349 -0
- data/Rakefile +38 -0
- data/bin/bluepill +124 -0
- data/bin/bpsv +3 -0
- data/bin/sample_forking_server +53 -0
- data/bluepill.gemspec +37 -0
- data/examples/example.rb +87 -0
- data/examples/new_example.rb +89 -0
- data/examples/new_runit_example.rb +29 -0
- data/examples/runit_example.rb +26 -0
- data/lib/bluepill.rb +38 -0
- data/lib/bluepill/application.rb +215 -0
- data/lib/bluepill/application/client.rb +8 -0
- data/lib/bluepill/application/server.rb +23 -0
- data/lib/bluepill/condition_watch.rb +51 -0
- data/lib/bluepill/controller.rb +122 -0
- data/lib/bluepill/dsl.rb +12 -0
- data/lib/bluepill/dsl/app_proxy.rb +25 -0
- data/lib/bluepill/dsl/process_factory.rb +122 -0
- data/lib/bluepill/dsl/process_proxy.rb +44 -0
- data/lib/bluepill/group.rb +72 -0
- data/lib/bluepill/logger.rb +63 -0
- data/lib/bluepill/process.rb +514 -0
- data/lib/bluepill/process_conditions.rb +14 -0
- data/lib/bluepill/process_conditions/always_true.rb +18 -0
- data/lib/bluepill/process_conditions/cpu_usage.rb +19 -0
- data/lib/bluepill/process_conditions/file_time.rb +26 -0
- data/lib/bluepill/process_conditions/http.rb +58 -0
- data/lib/bluepill/process_conditions/mem_usage.rb +32 -0
- data/lib/bluepill/process_conditions/process_condition.rb +22 -0
- data/lib/bluepill/process_journal.rb +219 -0
- data/lib/bluepill/process_statistics.rb +27 -0
- data/lib/bluepill/socket.rb +58 -0
- data/lib/bluepill/system.rb +265 -0
- data/lib/bluepill/trigger.rb +60 -0
- data/lib/bluepill/triggers/flapping.rb +56 -0
- data/lib/bluepill/util/rotational_array.rb +20 -0
- data/lib/bluepill/version.rb +4 -0
- data/local-bluepill +129 -0
- data/spec/lib/bluepill/logger_spec.rb +3 -0
- data/spec/lib/bluepill/process_spec.rb +96 -0
- data/spec/lib/bluepill/process_statistics_spec.rb +24 -0
- data/spec/lib/bluepill/system_spec.rb +36 -0
- data/spec/spec_helper.rb +15 -0
- metadata +302 -0
data/lib/bluepill.rb
ADDED
@@ -0,0 +1,38 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
require 'rubygems'
|
3
|
+
|
4
|
+
require 'bundler/setup' if ENV['BUNDLE_GEMFILE'] && File.exists?(ENV['BUNDLE_GEMFILE'])
|
5
|
+
|
6
|
+
require 'thread'
|
7
|
+
require 'monitor'
|
8
|
+
require 'syslog'
|
9
|
+
require 'timeout'
|
10
|
+
require 'logger'
|
11
|
+
|
12
|
+
require 'active_support/inflector'
|
13
|
+
require 'active_support/core_ext/hash'
|
14
|
+
require 'active_support/core_ext/numeric'
|
15
|
+
require 'active_support/duration'
|
16
|
+
|
17
|
+
require 'bluepill/dsl/process_proxy'
|
18
|
+
require 'bluepill/dsl/process_factory'
|
19
|
+
require 'bluepill/dsl/app_proxy'
|
20
|
+
|
21
|
+
require 'bluepill/application'
|
22
|
+
require 'bluepill/controller'
|
23
|
+
require 'bluepill/socket'
|
24
|
+
require "bluepill/process"
|
25
|
+
require "bluepill/process_statistics"
|
26
|
+
require "bluepill/group"
|
27
|
+
require "bluepill/logger"
|
28
|
+
require "bluepill/condition_watch"
|
29
|
+
require 'bluepill/trigger'
|
30
|
+
require 'bluepill/triggers/flapping'
|
31
|
+
require "bluepill/dsl"
|
32
|
+
require "bluepill/system"
|
33
|
+
|
34
|
+
require "bluepill/process_conditions"
|
35
|
+
|
36
|
+
require "bluepill/util/rotational_array"
|
37
|
+
|
38
|
+
require "bluepill/version"
|
@@ -0,0 +1,215 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
require 'thread'
|
3
|
+
require 'bluepill/system'
|
4
|
+
require 'bluepill/process_journal'
|
5
|
+
|
6
|
+
module Bluepill
|
7
|
+
class Application
|
8
|
+
PROCESS_COMMANDS = [:start, :stop, :restart, :unmonitor, :status]
|
9
|
+
|
10
|
+
attr_accessor :name, :logger, :base_dir, :socket, :pid_file, :kill_timeout
|
11
|
+
attr_accessor :groups, :work_queue
|
12
|
+
attr_accessor :pids_dir, :log_file
|
13
|
+
|
14
|
+
def initialize(name, options = {})
|
15
|
+
self.name = name
|
16
|
+
|
17
|
+
@foreground = options[:foreground]
|
18
|
+
self.log_file = options[:log_file]
|
19
|
+
self.base_dir = ProcessJournal.base_dir = options[:base_dir] || '/var/run/bluepill'
|
20
|
+
self.pid_file = File.join(self.base_dir, 'pids', self.name + ".pid")
|
21
|
+
self.pids_dir = File.join(self.base_dir, 'pids', self.name)
|
22
|
+
self.kill_timeout = options[:kill_timeout] || 10
|
23
|
+
|
24
|
+
self.groups = {}
|
25
|
+
|
26
|
+
self.logger = ProcessJournal.logger = Bluepill::Logger.new(:log_file => self.log_file, :stdout => foreground?).prefix_with(self.name)
|
27
|
+
|
28
|
+
self.setup_signal_traps
|
29
|
+
self.setup_pids_dir
|
30
|
+
|
31
|
+
@mutex = Mutex.new
|
32
|
+
end
|
33
|
+
|
34
|
+
def foreground?
|
35
|
+
!!@foreground
|
36
|
+
end
|
37
|
+
|
38
|
+
def mutex(&b)
|
39
|
+
@mutex.synchronize(&b)
|
40
|
+
end
|
41
|
+
|
42
|
+
def load
|
43
|
+
begin
|
44
|
+
self.start_server
|
45
|
+
rescue StandardError => e
|
46
|
+
$stderr.puts "Failed to start bluepill:"
|
47
|
+
$stderr.puts "%s `%s`" % [e.class.name, e.message]
|
48
|
+
$stderr.puts e.backtrace
|
49
|
+
exit(5)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
PROCESS_COMMANDS.each do |command|
|
54
|
+
class_eval <<-END
|
55
|
+
def #{command}(group_name = nil, process_name = nil)
|
56
|
+
self.send_to_process_or_group(:#{command}, group_name, process_name)
|
57
|
+
end
|
58
|
+
END
|
59
|
+
end
|
60
|
+
|
61
|
+
def add_process(process, group_name = nil)
|
62
|
+
group_name = group_name.to_s if group_name
|
63
|
+
|
64
|
+
self.groups[group_name] ||= Group.new(group_name, :logger => self.logger.prefix_with(group_name))
|
65
|
+
self.groups[group_name].add_process(process)
|
66
|
+
end
|
67
|
+
|
68
|
+
def version
|
69
|
+
Bluepill::VERSION
|
70
|
+
end
|
71
|
+
|
72
|
+
protected
|
73
|
+
def send_to_process_or_group(method, group_name, process_name)
|
74
|
+
if group_name.nil? && process_name.nil?
|
75
|
+
self.groups.values.collect do |group|
|
76
|
+
group.send(method)
|
77
|
+
end.flatten
|
78
|
+
elsif self.groups.key?(group_name)
|
79
|
+
self.groups[group_name].send(method, process_name)
|
80
|
+
elsif process_name.nil?
|
81
|
+
# they must be targeting just by process name
|
82
|
+
process_name = group_name
|
83
|
+
self.groups.values.collect do |group|
|
84
|
+
group.send(method, process_name)
|
85
|
+
end.flatten
|
86
|
+
else
|
87
|
+
[]
|
88
|
+
end
|
89
|
+
end
|
90
|
+
|
91
|
+
def start_listener
|
92
|
+
@listener_thread.kill if @listener_thread
|
93
|
+
@listener_thread = Thread.new do
|
94
|
+
loop do
|
95
|
+
begin
|
96
|
+
client = self.socket.accept
|
97
|
+
client.close_on_exec = true if client.respond_to?(:close_on_exec=)
|
98
|
+
command, *args = client.readline.strip.split(":")
|
99
|
+
response = begin
|
100
|
+
mutex { self.send(command, *args) }
|
101
|
+
rescue Exception => e
|
102
|
+
e
|
103
|
+
end
|
104
|
+
client.write(Marshal.dump(response))
|
105
|
+
rescue StandardError => e
|
106
|
+
logger.err("Got exception in cmd listener: %s `%s`" % [e.class.name, e.message])
|
107
|
+
e.backtrace.each {|l| logger.err(l)}
|
108
|
+
ensure
|
109
|
+
begin
|
110
|
+
client.close
|
111
|
+
rescue IOError
|
112
|
+
# closed stream
|
113
|
+
end
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
117
|
+
end
|
118
|
+
|
119
|
+
def start_server
|
120
|
+
self.kill_previous_bluepill
|
121
|
+
ProcessJournal.kill_all_from_journal(self.name)
|
122
|
+
ProcessJournal.clear_all_atomic_fs_locks(self.name)
|
123
|
+
::Process.setpgid(0, 0) rescue Errno::EPERM nil
|
124
|
+
|
125
|
+
Daemonize.daemonize unless foreground?
|
126
|
+
|
127
|
+
self.logger.reopen
|
128
|
+
|
129
|
+
$0 = "bluepilld: #{self.name}"
|
130
|
+
|
131
|
+
self.groups.each {|_, group| group.determine_initial_state }
|
132
|
+
|
133
|
+
|
134
|
+
self.write_pid_file
|
135
|
+
self.socket = Bluepill::Socket.server(self.base_dir, self.name)
|
136
|
+
self.start_listener
|
137
|
+
|
138
|
+
self.run
|
139
|
+
end
|
140
|
+
|
141
|
+
def run
|
142
|
+
@running = true # set to false by signal trap
|
143
|
+
while @running
|
144
|
+
mutex do
|
145
|
+
System.reset_data
|
146
|
+
self.groups.each { |_, group| group.tick }
|
147
|
+
end
|
148
|
+
sleep 1
|
149
|
+
end
|
150
|
+
end
|
151
|
+
|
152
|
+
def cleanup
|
153
|
+
ProcessJournal.kill_all_from_journal(self.name)
|
154
|
+
ProcessJournal.clear_all_atomic_fs_locks(self.name)
|
155
|
+
begin
|
156
|
+
System.delete_if_exists(self.socket.path) if self.socket
|
157
|
+
rescue IOError
|
158
|
+
end
|
159
|
+
System.delete_if_exists(self.pid_file)
|
160
|
+
end
|
161
|
+
|
162
|
+
def setup_signal_traps
|
163
|
+
terminator = Proc.new do
|
164
|
+
puts "Terminating..."
|
165
|
+
cleanup
|
166
|
+
@running = false
|
167
|
+
end
|
168
|
+
|
169
|
+
Signal.trap("TERM", &terminator)
|
170
|
+
Signal.trap("INT", &terminator)
|
171
|
+
|
172
|
+
Signal.trap("HUP") do
|
173
|
+
self.logger.reopen if self.logger
|
174
|
+
end
|
175
|
+
end
|
176
|
+
|
177
|
+
def setup_pids_dir
|
178
|
+
FileUtils.mkdir_p(self.pids_dir) unless File.exists?(self.pids_dir)
|
179
|
+
# we need everybody to be able to write to the pids_dir as processes managed by
|
180
|
+
# bluepill will be writing to this dir after they've dropped privileges
|
181
|
+
FileUtils.chmod(0777, self.pids_dir)
|
182
|
+
end
|
183
|
+
|
184
|
+
def kill_previous_bluepill
|
185
|
+
if File.exists?(self.pid_file)
|
186
|
+
previous_pid = File.read(self.pid_file).to_i
|
187
|
+
if System.pid_alive?(previous_pid)
|
188
|
+
begin
|
189
|
+
::Process.kill(0, previous_pid)
|
190
|
+
puts "Killing previous bluepilld[#{previous_pid}]"
|
191
|
+
::Process.kill(2, previous_pid)
|
192
|
+
rescue Exception => e
|
193
|
+
$stderr.puts "Encountered error trying to kill previous bluepill:"
|
194
|
+
$stderr.puts "#{e.class}: #{e.message}"
|
195
|
+
exit(4) unless e.is_a?(Errno::ESRCH)
|
196
|
+
else
|
197
|
+
kill_timeout.times do |i|
|
198
|
+
sleep 0.5
|
199
|
+
break unless System.pid_alive?(previous_pid)
|
200
|
+
end
|
201
|
+
|
202
|
+
if System.pid_alive?(previous_pid)
|
203
|
+
$stderr.puts "Previous bluepilld[#{previous_pid}] didn't die"
|
204
|
+
exit(4)
|
205
|
+
end
|
206
|
+
end
|
207
|
+
end
|
208
|
+
end
|
209
|
+
end
|
210
|
+
|
211
|
+
def write_pid_file
|
212
|
+
File.open(self.pid_file, 'w') { |x| x.write(::Process.pid) }
|
213
|
+
end
|
214
|
+
end
|
215
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
module Bluepill
|
3
|
+
module Application
|
4
|
+
module ServerMethods
|
5
|
+
|
6
|
+
def status
|
7
|
+
self.processes.collect do |process|
|
8
|
+
"#{process.name} #{process.state}"
|
9
|
+
end.join("\n")
|
10
|
+
end
|
11
|
+
|
12
|
+
def restart
|
13
|
+
self.socket = Bluepill::Socket.new(name, base_dir).client
|
14
|
+
socket.send("restart\n", 0)
|
15
|
+
end
|
16
|
+
|
17
|
+
def stop
|
18
|
+
self.socket = Bluepill::Socket.new(name, base_dir).client
|
19
|
+
socket.send("stop\n", 0)
|
20
|
+
end
|
21
|
+
end
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,51 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
module Bluepill
|
3
|
+
class HistoryValue < Struct.new(:value, :critical)
|
4
|
+
end
|
5
|
+
|
6
|
+
class ConditionWatch
|
7
|
+
attr_accessor :logger, :name
|
8
|
+
EMPTY_ARRAY = [].freeze # no need to recreate one every tick
|
9
|
+
|
10
|
+
def initialize(name, options = {})
|
11
|
+
@name = name
|
12
|
+
|
13
|
+
@logger = options.delete(:logger)
|
14
|
+
@fires = options.has_key?(:fires) ? Array(options.delete(:fires)) : [:restart]
|
15
|
+
@every = options.delete(:every)
|
16
|
+
@times = options.delete(:times) || [1,1]
|
17
|
+
@times = [@times, @times] unless @times.is_a?(Array) # handles :times => 5
|
18
|
+
@include_children = options.delete(:include_children) || false
|
19
|
+
|
20
|
+
self.clear_history!
|
21
|
+
|
22
|
+
@process_condition = ProcessConditions[@name].new(options)
|
23
|
+
end
|
24
|
+
|
25
|
+
def run(pid, tick_number = Time.now.to_i)
|
26
|
+
if @last_ran_at.nil? || (@last_ran_at + @every) <= tick_number
|
27
|
+
@last_ran_at = tick_number
|
28
|
+
|
29
|
+
value = @process_condition.run(pid, @include_children)
|
30
|
+
@history << HistoryValue.new(@process_condition.format_value(value), @process_condition.check(value))
|
31
|
+
self.logger.info(self.to_s)
|
32
|
+
|
33
|
+
return @fires if self.fired?
|
34
|
+
end
|
35
|
+
EMPTY_ARRAY
|
36
|
+
end
|
37
|
+
|
38
|
+
def clear_history!
|
39
|
+
@history = Util::RotationalArray.new(@times.last)
|
40
|
+
end
|
41
|
+
|
42
|
+
def fired?
|
43
|
+
@history.count {|v| not v.critical} >= @times.first
|
44
|
+
end
|
45
|
+
|
46
|
+
def to_s
|
47
|
+
data = @history.collect {|v| "#{v.value}#{'*' unless v.critical}"}.join(", ")
|
48
|
+
"#{@name}: [#{data}]\n"
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
@@ -0,0 +1,122 @@
|
|
1
|
+
# -*- encoding: utf-8 -*-
|
2
|
+
require 'fileutils'
|
3
|
+
require 'bluepill/system'
|
4
|
+
|
5
|
+
module Bluepill
|
6
|
+
class Controller
|
7
|
+
attr_accessor :base_dir, :log_file, :sockets_dir, :pids_dir
|
8
|
+
|
9
|
+
def initialize(options = {})
|
10
|
+
self.log_file = options[:log_file]
|
11
|
+
self.base_dir = options[:base_dir]
|
12
|
+
self.sockets_dir = File.join(base_dir, 'socks')
|
13
|
+
self.pids_dir = File.join(base_dir, 'pids')
|
14
|
+
|
15
|
+
setup_dir_structure
|
16
|
+
cleanup_bluepill_directory
|
17
|
+
end
|
18
|
+
|
19
|
+
def running_applications
|
20
|
+
Dir[File.join(sockets_dir, "*.sock")].map{|x| File.basename(x, ".sock")}
|
21
|
+
end
|
22
|
+
|
23
|
+
def handle_command(application, command, *args)
|
24
|
+
case command.to_sym
|
25
|
+
when :status
|
26
|
+
puts self.send_to_daemon(application, :status, *args)
|
27
|
+
when *Application::PROCESS_COMMANDS
|
28
|
+
# these need to be sent to the daemon and the results printed out
|
29
|
+
affected = self.send_to_daemon(application, command, *args)
|
30
|
+
if affected.empty?
|
31
|
+
puts "No processes effected"
|
32
|
+
else
|
33
|
+
puts "Sent #{command} to:"
|
34
|
+
affected.each do |process|
|
35
|
+
puts " #{process}"
|
36
|
+
end
|
37
|
+
end
|
38
|
+
when :quit
|
39
|
+
pid = pid_for(application)
|
40
|
+
if System.pid_alive?(pid)
|
41
|
+
::Process.kill("TERM", pid)
|
42
|
+
puts "Killing bluepilld[#{pid}]"
|
43
|
+
else
|
44
|
+
puts "bluepilld[#{pid}] not running"
|
45
|
+
end
|
46
|
+
when :log
|
47
|
+
log_file_location = self.send_to_daemon(application, :log_file)
|
48
|
+
log_file_location = self.log_file if log_file_location.to_s.strip.empty?
|
49
|
+
|
50
|
+
requested_pattern = args.first
|
51
|
+
grep_pattern = self.grep_pattern(application, requested_pattern)
|
52
|
+
|
53
|
+
tail = "tail -n 100 -f #{log_file_location} | grep -E '#{grep_pattern}'"
|
54
|
+
puts "Tailing log for #{requested_pattern}..."
|
55
|
+
Kernel.exec(tail)
|
56
|
+
else
|
57
|
+
$stderr.puts "Unknown command `%s` (or application `%s` has not been loaded yet)" % [command, command]
|
58
|
+
exit(1)
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
def send_to_daemon(application, command, *args)
|
63
|
+
begin
|
64
|
+
verify_version!(application)
|
65
|
+
|
66
|
+
command = ([command, *args]).join(":")
|
67
|
+
response = Socket.client_command(base_dir, application, command)
|
68
|
+
if response.is_a?(Exception)
|
69
|
+
$stderr.puts "Received error from server:"
|
70
|
+
$stderr.puts response.inspect
|
71
|
+
$stderr.puts response.backtrace.join("\n")
|
72
|
+
exit(8)
|
73
|
+
else
|
74
|
+
response
|
75
|
+
end
|
76
|
+
|
77
|
+
rescue Errno::ECONNREFUSED
|
78
|
+
abort("Connection Refused: Server is not running")
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
def grep_pattern(application, query = nil)
|
83
|
+
pattern = [application, query].compact.join(':')
|
84
|
+
['\[.*', Regexp.escape(pattern), '.*'].compact.join
|
85
|
+
end
|
86
|
+
private
|
87
|
+
|
88
|
+
def cleanup_bluepill_directory
|
89
|
+
self.running_applications.each do |app|
|
90
|
+
pid = pid_for(app)
|
91
|
+
if !pid || !System.pid_alive?(pid)
|
92
|
+
pid_file = File.join(self.pids_dir, "#{app}.pid")
|
93
|
+
sock_file = File.join(self.sockets_dir, "#{app}.sock")
|
94
|
+
System.delete_if_exists(pid_file)
|
95
|
+
System.delete_if_exists(sock_file)
|
96
|
+
end
|
97
|
+
end
|
98
|
+
end
|
99
|
+
|
100
|
+
def pid_for(app)
|
101
|
+
pid_file = File.join(self.pids_dir, "#{app}.pid")
|
102
|
+
File.exists?(pid_file) && File.read(pid_file).to_i
|
103
|
+
end
|
104
|
+
|
105
|
+
def setup_dir_structure
|
106
|
+
[@sockets_dir, @pids_dir].each do |dir|
|
107
|
+
FileUtils.mkdir_p(dir) unless File.exists?(dir)
|
108
|
+
end
|
109
|
+
end
|
110
|
+
|
111
|
+
def verify_version!(application)
|
112
|
+
begin
|
113
|
+
version = Socket.client_command(base_dir, application, "version")
|
114
|
+
if version != Bluepill::VERSION
|
115
|
+
abort("The running version of your daemon seems to be out of date.\nDaemon Version: #{version}, CLI Version: #{Bluepill::VERSION}")
|
116
|
+
end
|
117
|
+
rescue ArgumentError
|
118
|
+
abort("The running version of your daemon seems to be out of date.")
|
119
|
+
end
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|