bluepill 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/CHANGELOG.md +835 -0
- data/CONTRIBUTING.md +31 -0
- data/LICENSE +1 -1
- data/README.md +66 -29
- data/bin/bluepill +1 -1
- data/bluepill.gemspec +17 -6
- data/lib/bluepill/application.rb +321 -46
- data/lib/bluepill/condition_watch.rb +103 -8
- data/lib/bluepill/controller.rb +124 -13
- data/lib/bluepill/dsl/app_proxy.rb +102 -3
- data/lib/bluepill/dsl/process_factory.rb +183 -60
- data/lib/bluepill/dsl/process_proxy.rb +132 -14
- data/lib/bluepill/dsl.rb +27 -5
- data/lib/bluepill/group.rb +142 -9
- data/lib/bluepill/logger.rb +122 -5
- data/lib/bluepill/process.rb +575 -201
- data/lib/bluepill/process_commands.rb +264 -0
- data/lib/bluepill/process_conditions/always_true.rb +18 -4
- data/lib/bluepill/process_conditions/cpu_usage.rb +34 -0
- data/lib/bluepill/process_conditions/file_time.rb +40 -4
- data/lib/bluepill/process_conditions/http.rb +81 -23
- data/lib/bluepill/process_conditions/mem_usage.rb +61 -2
- data/lib/bluepill/process_conditions/process_condition.rb +43 -0
- data/lib/bluepill/process_conditions/running_time.rb +36 -1
- data/lib/bluepill/process_conditions/zombie_process.rb +26 -5
- data/lib/bluepill/process_conditions.rb +15 -1
- data/lib/bluepill/process_journal.rb +374 -114
- data/lib/bluepill/process_statistics.rb +33 -3
- data/lib/bluepill/socket.rb +63 -16
- data/lib/bluepill/system/commands.rb +286 -0
- data/lib/bluepill/system.rb +223 -228
- data/lib/bluepill/trigger.rb +110 -14
- data/lib/bluepill/triggers/flapping.rb +84 -3
- data/lib/bluepill/util/rotational_array.rb +24 -1
- data/lib/bluepill/version.rb +71 -19
- data/lib/bluepill.rb +25 -2
- data/sig/bluepill.rbs +941 -0
- data/sig/manifest.yaml +7 -0
- metadata +16 -34
- data/bin/sample_forking_server +0 -66
- data/lib/bluepill/application/client.rb +0 -6
- data/lib/bluepill/application/server.rb +0 -21
data/lib/bluepill/application.rb
CHANGED
|
@@ -3,43 +3,154 @@ require 'bluepill/system'
|
|
|
3
3
|
require 'bluepill/process_journal'
|
|
4
4
|
|
|
5
5
|
module Bluepill
|
|
6
|
+
# The bluepill daemon for one application
|
|
7
|
+
#
|
|
8
|
+
# It holds the application's groups of processes, ticks them every
|
|
9
|
+
# second, and answers the commands that the CLI sends over its socket.
|
|
10
|
+
#
|
|
11
|
+
# @api private
|
|
6
12
|
class Application
|
|
13
|
+
# The commands that act on a process or group, and that the CLI sends
|
|
7
14
|
PROCESS_COMMANDS = [:start, :stop, :restart, :unmonitor, :status].freeze
|
|
8
15
|
|
|
9
|
-
attr_accessor :name, :logger, :base_dir, :socket, :pid_file, :kill_timeout
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
16
|
+
attr_accessor :name, :logger, :base_dir, :socket, :pid_file, :kill_timeout, :groups, :work_queue, :pids_dir, :log_file
|
|
17
|
+
|
|
18
|
+
# @!attribute [rw] name
|
|
19
|
+
# The application's name, which names its socket and pid file
|
|
20
|
+
#
|
|
21
|
+
# @return [String]
|
|
22
|
+
#
|
|
23
|
+
# @api private
|
|
24
|
+
|
|
25
|
+
# @!attribute [rw] logger
|
|
26
|
+
# The application's logger, prefixed with its name
|
|
27
|
+
#
|
|
28
|
+
# @return [Logger]
|
|
29
|
+
#
|
|
30
|
+
# @api private
|
|
31
|
+
|
|
32
|
+
# @!attribute [rw] base_dir
|
|
33
|
+
# The directory with the daemon's sockets, pid files and journals
|
|
34
|
+
#
|
|
35
|
+
# @return [String]
|
|
36
|
+
#
|
|
37
|
+
# @api private
|
|
38
|
+
|
|
39
|
+
# @!attribute [rw] socket
|
|
40
|
+
# The socket the daemon listens for commands on, once it's started
|
|
41
|
+
#
|
|
42
|
+
# @return [UNIXServer, nil]
|
|
43
|
+
#
|
|
44
|
+
# @api private
|
|
45
|
+
|
|
46
|
+
# @!attribute [rw] pid_file
|
|
47
|
+
# The daemon's own pid file
|
|
48
|
+
#
|
|
49
|
+
# @return [String]
|
|
50
|
+
#
|
|
51
|
+
# @api private
|
|
52
|
+
|
|
53
|
+
# @!attribute [rw] kill_timeout
|
|
54
|
+
# How many half seconds to wait for a previous daemon to exit
|
|
55
|
+
#
|
|
56
|
+
# @return [Integer]
|
|
57
|
+
#
|
|
58
|
+
# @api private
|
|
59
|
+
|
|
60
|
+
# @!attribute [rw] groups
|
|
61
|
+
# The application's groups by name, with nil for processes in no group
|
|
62
|
+
#
|
|
63
|
+
# @return [Hash{String, nil => Group}]
|
|
64
|
+
#
|
|
65
|
+
# @api private
|
|
66
|
+
|
|
67
|
+
# @!attribute [rw] work_queue
|
|
68
|
+
# An attribute that bluepill doesn't use
|
|
69
|
+
#
|
|
70
|
+
# @return [Object]
|
|
71
|
+
#
|
|
72
|
+
# @api private
|
|
73
|
+
|
|
74
|
+
# @!attribute [rw] pids_dir
|
|
75
|
+
# The directory with the pid files of the application's processes
|
|
76
|
+
#
|
|
77
|
+
# @return [String]
|
|
78
|
+
#
|
|
79
|
+
# @api private
|
|
80
|
+
|
|
81
|
+
# @!attribute [rw] log_file
|
|
82
|
+
# The file the daemon logs to, or nil to log to syslog
|
|
83
|
+
#
|
|
84
|
+
# @return [String, nil]
|
|
85
|
+
#
|
|
86
|
+
# @api private
|
|
87
|
+
|
|
88
|
+
# Create the daemon for an application
|
|
89
|
+
#
|
|
90
|
+
# It sets up signal traps, the logger and the pids directory. It exits
|
|
91
|
+
# with status 5 if it can't write to its log file or base directory.
|
|
92
|
+
#
|
|
93
|
+
# @param [String] name the application's name
|
|
94
|
+
# @param [Hash{Symbol => Object}] options :foreground, :log_file,
|
|
95
|
+
# :base_dir and :kill_timeout
|
|
96
|
+
#
|
|
97
|
+
# @return [void]
|
|
98
|
+
#
|
|
99
|
+
# @api private
|
|
13
100
|
def initialize(name, options = {})
|
|
14
101
|
self.name = name
|
|
15
102
|
|
|
16
103
|
@foreground = options[:foreground]
|
|
17
104
|
self.log_file = options[:log_file]
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
105
|
+
# Only used for a non-root user, who has a home
|
|
106
|
+
home = ENV['HOME'] #: String
|
|
107
|
+
self.base_dir = options[:base_dir] ||
|
|
108
|
+
ENV['BLUEPILL_BASE_DIR'] ||
|
|
109
|
+
(::Process.euid.zero? ? '/var/run/bluepill' : File.join(home, '.bluepill'))
|
|
110
|
+
ProcessJournal.setup(base_dir, name)
|
|
111
|
+
self.pid_file = File.join(base_dir, 'pids', "#{name}.pid")
|
|
112
|
+
self.pids_dir = File.join(base_dir, 'pids', name)
|
|
23
113
|
self.kill_timeout = options[:kill_timeout] || 10
|
|
24
114
|
|
|
25
115
|
self.groups = {}
|
|
26
116
|
|
|
27
|
-
self.logger = ProcessJournal.logger = Bluepill::Logger.new(log_file: log_file, stdout: foreground?).prefix_with(self.name)
|
|
28
|
-
|
|
29
117
|
setup_signal_traps
|
|
30
|
-
|
|
118
|
+
setup_logger_and_pids_dir
|
|
31
119
|
|
|
32
120
|
@mutex = Mutex.new
|
|
33
121
|
end
|
|
34
122
|
|
|
123
|
+
# Return whether the daemon runs in the foreground
|
|
124
|
+
#
|
|
125
|
+
# @return [Boolean]
|
|
126
|
+
#
|
|
127
|
+
# @api private
|
|
35
128
|
def foreground?
|
|
36
129
|
!!@foreground
|
|
37
130
|
end
|
|
38
131
|
|
|
39
|
-
|
|
40
|
-
|
|
132
|
+
# Run a block while holding the daemon's lock
|
|
133
|
+
#
|
|
134
|
+
# Ticks and commands from the CLI hold it, so they never overlap.
|
|
135
|
+
#
|
|
136
|
+
# @yield the work to do while holding the lock
|
|
137
|
+
# @yieldreturn [Object] what the block returns
|
|
138
|
+
#
|
|
139
|
+
# @return [Object] what the block returns
|
|
140
|
+
#
|
|
141
|
+
# @api private
|
|
142
|
+
def mutex(&block)
|
|
143
|
+
@mutex.synchronize(&block)
|
|
41
144
|
end
|
|
42
145
|
|
|
146
|
+
# Start the daemon and monitor the application's processes
|
|
147
|
+
#
|
|
148
|
+
# It prints what went wrong, and exits with status 5, if the daemon
|
|
149
|
+
# fails to start.
|
|
150
|
+
#
|
|
151
|
+
# @return [void]
|
|
152
|
+
#
|
|
153
|
+
# @api private
|
|
43
154
|
def load
|
|
44
155
|
start_server
|
|
45
156
|
rescue StandardError => e
|
|
@@ -49,65 +160,157 @@ module Bluepill
|
|
|
49
160
|
exit(5)
|
|
50
161
|
end
|
|
51
162
|
|
|
163
|
+
# @!method start(group_name = nil, process_name = nil)
|
|
164
|
+
# Start a process or group, or every process without either
|
|
165
|
+
#
|
|
166
|
+
# @param [String, nil] group_name the group, or the process's name
|
|
167
|
+
# when no group has it
|
|
168
|
+
# @param [String, nil] process_name the process
|
|
169
|
+
#
|
|
170
|
+
# @return [Array<String>] the affected processes, as group:process
|
|
171
|
+
#
|
|
172
|
+
# @api private
|
|
173
|
+
|
|
174
|
+
# @!method stop(group_name = nil, process_name = nil)
|
|
175
|
+
# Stop a process or group, or every process without either
|
|
176
|
+
#
|
|
177
|
+
# @param [String, nil] group_name the group, or the process's name
|
|
178
|
+
# when no group has it
|
|
179
|
+
# @param [String, nil] process_name the process
|
|
180
|
+
#
|
|
181
|
+
# @return [Array<String>] the affected processes, as group:process
|
|
182
|
+
#
|
|
183
|
+
# @api private
|
|
184
|
+
|
|
185
|
+
# @!method restart(group_name = nil, process_name = nil)
|
|
186
|
+
# Restart a process or group, or every process without either
|
|
187
|
+
#
|
|
188
|
+
# @param [String, nil] group_name the group, or the process's name
|
|
189
|
+
# when no group has it
|
|
190
|
+
# @param [String, nil] process_name the process
|
|
191
|
+
#
|
|
192
|
+
# @return [Array<String>] the affected processes, as group:process
|
|
193
|
+
#
|
|
194
|
+
# @api private
|
|
195
|
+
|
|
196
|
+
# @!method unmonitor(group_name = nil, process_name = nil)
|
|
197
|
+
# Stop monitoring a process or group, or every process without either
|
|
198
|
+
#
|
|
199
|
+
# @param [String, nil] group_name the group, or the process's name
|
|
200
|
+
# when no group has it
|
|
201
|
+
# @param [String, nil] process_name the process
|
|
202
|
+
#
|
|
203
|
+
# @return [Array<String>] the affected processes, as group:process
|
|
204
|
+
#
|
|
205
|
+
# @api private
|
|
206
|
+
|
|
207
|
+
# @!method status(group_name = nil, process_name = nil)
|
|
208
|
+
# Return the status of a process or group, or of every process
|
|
209
|
+
#
|
|
210
|
+
# @param [String, nil] group_name the group, or the process's name
|
|
211
|
+
# when no group has it
|
|
212
|
+
# @param [String, nil] process_name the process
|
|
213
|
+
#
|
|
214
|
+
# @return [Array<String>] the status lines
|
|
215
|
+
#
|
|
216
|
+
# @api private
|
|
217
|
+
|
|
218
|
+
# Define a method for each of the PROCESS_COMMANDS
|
|
52
219
|
PROCESS_COMMANDS.each do |command|
|
|
53
|
-
class_eval <<-
|
|
220
|
+
class_eval <<-RUBY, __FILE__, __LINE__ + 1
|
|
54
221
|
def #{command}(group_name = nil, process_name = nil)
|
|
55
222
|
self.send_to_process_or_group(:#{command}, group_name, process_name)
|
|
56
223
|
end
|
|
57
|
-
|
|
224
|
+
RUBY
|
|
58
225
|
end
|
|
59
226
|
|
|
227
|
+
# Add a process to a group, which is created if it's new
|
|
228
|
+
#
|
|
229
|
+
# @param [Process] process the process
|
|
230
|
+
# @param [#to_s, nil] group_name the group, or nil for no group
|
|
231
|
+
#
|
|
232
|
+
# @return [Array<Process>] the group's processes
|
|
233
|
+
#
|
|
234
|
+
# @api private
|
|
60
235
|
def add_process(process, group_name = nil)
|
|
61
236
|
group_name = group_name.to_s if group_name
|
|
62
237
|
|
|
63
238
|
groups[group_name] ||= Group.new(group_name, logger: logger.prefix_with(group_name))
|
|
64
|
-
groups
|
|
239
|
+
groups.fetch(group_name).add_process(process)
|
|
65
240
|
end
|
|
66
241
|
|
|
242
|
+
# Return the daemon's version, which the CLI checks against its own
|
|
243
|
+
#
|
|
244
|
+
# @return [Module] the Version module
|
|
245
|
+
#
|
|
246
|
+
# @api private
|
|
67
247
|
def version
|
|
68
|
-
|
|
248
|
+
Version
|
|
69
249
|
end
|
|
70
250
|
|
|
71
251
|
protected
|
|
72
252
|
|
|
253
|
+
# Send a command to a process, a group, or every process
|
|
254
|
+
#
|
|
255
|
+
# @param [Symbol] method the command
|
|
256
|
+
# @param [String, nil] group_name the group, or the start of a process
|
|
257
|
+
# name that contains a colon
|
|
258
|
+
# @param [String, nil] process_name the process
|
|
259
|
+
#
|
|
260
|
+
# @return [Array<String>] what the groups returned, flattened
|
|
261
|
+
#
|
|
262
|
+
# @api private
|
|
73
263
|
def send_to_process_or_group(method, group_name, process_name)
|
|
74
264
|
if group_name.nil? && process_name.nil?
|
|
75
265
|
groups.values.collect do |group|
|
|
76
|
-
group.
|
|
266
|
+
group.public_send(method)
|
|
77
267
|
end.flatten
|
|
78
268
|
elsif groups.key?(group_name)
|
|
79
|
-
groups
|
|
80
|
-
|
|
81
|
-
# they must be targeting just by process name
|
|
82
|
-
process_name = group_name
|
|
269
|
+
groups.fetch(group_name).public_send(method, process_name)
|
|
270
|
+
else
|
|
271
|
+
# they must be targeting just by process name, which may contain ':'
|
|
272
|
+
process_name = [group_name, process_name].compact.join(':')
|
|
83
273
|
groups.values.collect do |group|
|
|
84
|
-
group.
|
|
274
|
+
group.public_send(method, process_name)
|
|
85
275
|
end.flatten
|
|
86
|
-
else
|
|
87
|
-
[]
|
|
88
276
|
end
|
|
89
277
|
end
|
|
90
278
|
|
|
279
|
+
# Start the thread that answers commands on the socket
|
|
280
|
+
#
|
|
281
|
+
# It kills the thread that a previous call started. Each command is a
|
|
282
|
+
# line, such as start:group:process, and the answer is the command's
|
|
283
|
+
# result, or the error it raised, marshaled.
|
|
284
|
+
#
|
|
285
|
+
# @return [Thread] the listener thread
|
|
286
|
+
#
|
|
287
|
+
# @api private
|
|
91
288
|
def start_listener
|
|
92
|
-
|
|
289
|
+
previous_thread = @listener_thread
|
|
290
|
+
previous_thread.kill if previous_thread
|
|
93
291
|
@listener_thread = Thread.new do
|
|
94
292
|
loop do
|
|
293
|
+
client = nil #: UNIXSocket?
|
|
95
294
|
begin
|
|
96
|
-
|
|
97
|
-
client
|
|
98
|
-
|
|
295
|
+
server = socket #: UNIXServer
|
|
296
|
+
client = server.accept
|
|
297
|
+
client.close_on_exec = true if client.respond_to?(:close_on_exec=)
|
|
298
|
+
# The target is group:process and the process name may contain ':'
|
|
299
|
+
command, *args = client.readline.strip.split(':', 3).reject(&:empty?)
|
|
99
300
|
response = begin
|
|
100
|
-
|
|
101
|
-
|
|
301
|
+
# An empty line leaves command nil, so public_send raises an error for the client
|
|
302
|
+
mutex { public_send(command, *args) } # steep:ignore ArgumentTypeMismatch
|
|
303
|
+
rescue StandardError => e
|
|
102
304
|
e
|
|
103
305
|
end
|
|
104
306
|
client.write(Marshal.dump(response))
|
|
105
307
|
rescue StandardError => e
|
|
106
308
|
logger.err(format('Got exception in cmd listener: %s `%s`', e.class.name, e.message))
|
|
107
|
-
e.backtrace.each { |l| logger.err(l) }
|
|
309
|
+
Array(e.backtrace).each { |l| logger.err(l) }
|
|
108
310
|
ensure
|
|
109
311
|
begin
|
|
110
|
-
|
|
312
|
+
# Ruby before 3.0 never counts branches in ensure as covered
|
|
313
|
+
client && client.close
|
|
111
314
|
rescue IOError
|
|
112
315
|
# closed stream
|
|
113
316
|
end
|
|
@@ -116,9 +319,19 @@ module Bluepill
|
|
|
116
319
|
end
|
|
117
320
|
end
|
|
118
321
|
|
|
322
|
+
# Start the daemon
|
|
323
|
+
#
|
|
324
|
+
# It stops a previous daemon for the application, clears journal locks
|
|
325
|
+
# it left behind, daemonizes unless it runs in the foreground, opens the
|
|
326
|
+
# socket, and runs until a signal stops it. Processes that are already
|
|
327
|
+
# running, from their pid files, are monitored again, not restarted, so
|
|
328
|
+
# bluepill can be upgraded without restarting them.
|
|
329
|
+
#
|
|
330
|
+
# @return [void]
|
|
331
|
+
#
|
|
332
|
+
# @api private
|
|
119
333
|
def start_server
|
|
120
334
|
kill_previous_bluepill
|
|
121
|
-
ProcessJournal.kill_all_from_all_journals
|
|
122
335
|
ProcessJournal.clear_all_atomic_fs_locks
|
|
123
336
|
|
|
124
337
|
begin
|
|
@@ -126,34 +339,49 @@ module Bluepill
|
|
|
126
339
|
rescue Errno::EPERM
|
|
127
340
|
end
|
|
128
341
|
|
|
129
|
-
|
|
342
|
+
unless foreground?
|
|
343
|
+
Daemonize.daemonize
|
|
344
|
+
# Daemonizing ignores HUP, so trap it again to reopen the log on HUP
|
|
345
|
+
setup_signal_traps
|
|
346
|
+
end
|
|
130
347
|
|
|
131
348
|
logger.reopen
|
|
132
349
|
|
|
133
350
|
$0 = "bluepilld: #{name}"
|
|
134
351
|
|
|
135
|
-
groups.
|
|
352
|
+
groups.each_value(&:determine_initial_state)
|
|
136
353
|
|
|
137
354
|
write_pid_file
|
|
138
|
-
self.socket =
|
|
355
|
+
self.socket = Socket.server(base_dir, name)
|
|
139
356
|
start_listener
|
|
140
357
|
|
|
141
358
|
run
|
|
142
359
|
end
|
|
143
360
|
|
|
361
|
+
# Tick every group once a second, until a signal stops the daemon
|
|
362
|
+
#
|
|
363
|
+
# @return [void]
|
|
364
|
+
#
|
|
365
|
+
# @api private
|
|
144
366
|
def run
|
|
145
367
|
@running = true # set to false by signal trap
|
|
146
368
|
while @running
|
|
147
369
|
mutex do
|
|
148
370
|
System.reset_data
|
|
149
|
-
groups.
|
|
371
|
+
groups.each_value(&:tick)
|
|
150
372
|
end
|
|
151
373
|
sleep 1
|
|
152
374
|
end
|
|
153
375
|
end
|
|
154
376
|
|
|
377
|
+
# Clear the app's journal locks, and remove the socket and pid file
|
|
378
|
+
#
|
|
379
|
+
# The processes keep running, so the next daemon monitors them again.
|
|
380
|
+
#
|
|
381
|
+
# @return [void]
|
|
382
|
+
#
|
|
383
|
+
# @api private
|
|
155
384
|
def cleanup
|
|
156
|
-
ProcessJournal.kill_all_from_all_journals
|
|
157
385
|
ProcessJournal.clear_all_atomic_fs_locks
|
|
158
386
|
begin
|
|
159
387
|
System.delete_if_exists(socket.path) if socket
|
|
@@ -162,12 +390,17 @@ module Bluepill
|
|
|
162
390
|
System.delete_if_exists(pid_file)
|
|
163
391
|
end
|
|
164
392
|
|
|
393
|
+
# Trap TERM and INT to clean up and stop, and HUP to reopen the log
|
|
394
|
+
#
|
|
395
|
+
# @return [void]
|
|
396
|
+
#
|
|
397
|
+
# @api private
|
|
165
398
|
def setup_signal_traps
|
|
166
399
|
terminator = proc do
|
|
167
400
|
puts 'Terminating...'
|
|
168
401
|
cleanup
|
|
169
402
|
@running = false
|
|
170
|
-
end
|
|
403
|
+
end #: ^(Integer signal) -> void
|
|
171
404
|
|
|
172
405
|
Signal.trap('TERM', &terminator)
|
|
173
406
|
Signal.trap('INT', &terminator)
|
|
@@ -177,24 +410,61 @@ module Bluepill
|
|
|
177
410
|
end
|
|
178
411
|
end
|
|
179
412
|
|
|
413
|
+
# Create the logger and the pids directory
|
|
414
|
+
#
|
|
415
|
+
# Bluepill opens its log file and creates its pids dir as the user that
|
|
416
|
+
# runs it. The uid and gid in the config only apply to its processes.
|
|
417
|
+
# It exits with status 5 if it can't.
|
|
418
|
+
#
|
|
419
|
+
# @return [void]
|
|
420
|
+
#
|
|
421
|
+
# @api private
|
|
422
|
+
def setup_logger_and_pids_dir
|
|
423
|
+
self.logger = ProcessJournal.logger = Logger.new(log_file: log_file, stdout: foreground?).prefix_with(name)
|
|
424
|
+
setup_pids_dir
|
|
425
|
+
rescue SystemCallError => e
|
|
426
|
+
$stderr.puts("Failed to start bluepill: #{e}")
|
|
427
|
+
message = 'The user running bluepill must be able to write to its log_file and base_dir. ' \
|
|
428
|
+
'The uid and gid in the config only apply to the processes it monitors.'
|
|
429
|
+
$stderr.puts(message)
|
|
430
|
+
exit(5)
|
|
431
|
+
end
|
|
432
|
+
|
|
433
|
+
# Create the pids directory, and let every user write to it
|
|
434
|
+
#
|
|
435
|
+
# @return [void]
|
|
436
|
+
#
|
|
437
|
+
# @api private
|
|
180
438
|
def setup_pids_dir
|
|
181
439
|
FileUtils.mkdir_p(pids_dir) unless File.exist?(pids_dir)
|
|
182
440
|
# we need everybody to be able to write to the pids_dir as processes managed by
|
|
183
441
|
# bluepill will be writing to this dir after they've dropped privileges
|
|
184
|
-
FileUtils.chmod(
|
|
442
|
+
FileUtils.chmod(0o777, pids_dir)
|
|
185
443
|
end
|
|
186
444
|
|
|
445
|
+
# Interrupt a daemon for this application that's still running
|
|
446
|
+
#
|
|
447
|
+
# It waits up to kill_timeout half seconds for the daemon to exit, and
|
|
448
|
+
# exits with status 4 if it doesn't, or if signaling it fails.
|
|
449
|
+
#
|
|
450
|
+
# @return [void]
|
|
451
|
+
#
|
|
452
|
+
# @api private
|
|
187
453
|
def kill_previous_bluepill
|
|
188
454
|
return unless File.exist?(pid_file)
|
|
455
|
+
|
|
456
|
+
# An empty or garbled pid file reads as 0, and signaling pid 0 would
|
|
457
|
+
# interrupt bluepill's own process group
|
|
189
458
|
previous_pid = File.read(pid_file).to_i
|
|
190
|
-
return unless System.pid_alive?(previous_pid)
|
|
459
|
+
return unless previous_pid > 0 && System.pid_alive?(previous_pid)
|
|
460
|
+
|
|
191
461
|
::Process.kill(0, previous_pid)
|
|
192
462
|
puts "Killing previous bluepilld[#{previous_pid}]"
|
|
193
463
|
::Process.kill(2, previous_pid)
|
|
194
|
-
rescue => e
|
|
464
|
+
rescue StandardError => e
|
|
195
465
|
$stderr.puts 'Encountered error trying to kill previous bluepill:'
|
|
196
|
-
$stderr.puts "#{e.class}: #{e
|
|
197
|
-
exit(4) unless e.
|
|
466
|
+
$stderr.puts "#{e.class}: #{e}"
|
|
467
|
+
exit(4) unless e.instance_of?(Errno::ESRCH)
|
|
198
468
|
else
|
|
199
469
|
kill_timeout.times do |_i|
|
|
200
470
|
sleep 0.5
|
|
@@ -207,6 +477,11 @@ module Bluepill
|
|
|
207
477
|
end
|
|
208
478
|
end
|
|
209
479
|
|
|
480
|
+
# Write the daemon's pid to its pid file
|
|
481
|
+
#
|
|
482
|
+
# @return [void]
|
|
483
|
+
#
|
|
484
|
+
# @api private
|
|
210
485
|
def write_pid_file
|
|
211
486
|
File.open(pid_file, 'w') { |x| x.write(::Process.pid) }
|
|
212
487
|
end
|
|
@@ -1,18 +1,76 @@
|
|
|
1
1
|
module Bluepill
|
|
2
|
+
# One entry of a watch's history: the value it read, as its log shows it,
|
|
3
|
+
# and whether the value was fine
|
|
4
|
+
#
|
|
5
|
+
# @!attribute value
|
|
6
|
+
# Return the value as the history shows it
|
|
7
|
+
#
|
|
8
|
+
# @return [#to_s] the formatted value
|
|
9
|
+
#
|
|
10
|
+
# @api private
|
|
11
|
+
#
|
|
12
|
+
# @!attribute critical
|
|
13
|
+
# Return whether the value was fine
|
|
14
|
+
#
|
|
15
|
+
# It's true for a value that passed the check, and false for one that
|
|
16
|
+
# counts toward the watch firing.
|
|
17
|
+
#
|
|
18
|
+
# @return [Boolean] whether the value passed the check
|
|
19
|
+
#
|
|
20
|
+
# @api private
|
|
21
|
+
#
|
|
22
|
+
# @api private
|
|
2
23
|
HistoryValue = Struct.new(:value, :critical)
|
|
3
24
|
|
|
25
|
+
# Runs a process's check every so often, and fires its events when the
|
|
26
|
+
# check fails often enough
|
|
27
|
+
#
|
|
28
|
+
# A config's `checks :mem_usage, every: 10.seconds, below: 100.megabytes,
|
|
29
|
+
# times: [3, 5]` makes one: it runs every 10 seconds, and fires (restarts
|
|
30
|
+
# the process, by default) when 3 of the last 5 values failed.
|
|
31
|
+
#
|
|
32
|
+
# @api private
|
|
4
33
|
class ConditionWatch
|
|
5
|
-
|
|
6
|
-
|
|
34
|
+
# Return the logger the watch writes its history to
|
|
35
|
+
#
|
|
36
|
+
# @return [Logger] the process's logger
|
|
37
|
+
#
|
|
38
|
+
# @api private
|
|
39
|
+
attr_accessor :logger
|
|
7
40
|
|
|
41
|
+
# Return the check's name, such as :mem_usage
|
|
42
|
+
#
|
|
43
|
+
# @return [Symbol] the name a config gave the check
|
|
44
|
+
#
|
|
45
|
+
# @api private
|
|
46
|
+
attr_accessor :name
|
|
47
|
+
|
|
48
|
+
no_events = [] #: Array[Symbol]
|
|
49
|
+
|
|
50
|
+
# The events of a run that fires nothing, made once instead of every tick
|
|
51
|
+
#
|
|
52
|
+
# @api private
|
|
53
|
+
EMPTY_ARRAY = no_events.freeze
|
|
54
|
+
|
|
55
|
+
# Create a watch of a check
|
|
56
|
+
#
|
|
57
|
+
# @param [Symbol] name the check's name, such as :mem_usage
|
|
58
|
+
# @param [Hash{Symbol => Object}] options the check's options. The
|
|
59
|
+
# watch takes :logger, :fires (the events to fire, [:restart] by
|
|
60
|
+
# default), :every (seconds between runs), :times (how many of the
|
|
61
|
+
# last values must fail, as a count or [count, of]), and
|
|
62
|
+
# :include_children. The check gets the rest.
|
|
63
|
+
#
|
|
64
|
+
# @return [void]
|
|
65
|
+
#
|
|
66
|
+
# @api private
|
|
8
67
|
def initialize(name, options = {})
|
|
9
68
|
@name = name
|
|
10
69
|
|
|
11
70
|
@logger = options.delete(:logger)
|
|
12
71
|
@fires = options.key?(:fires) ? Array(options.delete(:fires)) : [:restart]
|
|
13
72
|
@every = options.delete(:every)
|
|
14
|
-
@times = options.delete(:times) ||
|
|
15
|
-
@times = [@times, @times] unless @times.is_a?(Array) # handles times: 5
|
|
73
|
+
@times = Array(options.delete(:times) || 1) # handles times: 5
|
|
16
74
|
@include_children = options.delete(:include_children) || false
|
|
17
75
|
|
|
18
76
|
clear_history!
|
|
@@ -20,15 +78,35 @@ module Bluepill
|
|
|
20
78
|
@process_condition = ProcessConditions[@name].new(options)
|
|
21
79
|
end
|
|
22
80
|
|
|
23
|
-
|
|
24
|
-
|
|
81
|
+
# Run the check if it's due, and return the events to fire
|
|
82
|
+
#
|
|
83
|
+
# A check is due on its first run, and then every `every` seconds. A
|
|
84
|
+
# run adds the value to the history and logs the history.
|
|
85
|
+
#
|
|
86
|
+
# @param [Integer, nil] pid the process's pid
|
|
87
|
+
# @param [Integer] tick_number the current time, in seconds
|
|
88
|
+
#
|
|
89
|
+
# @return [Array<Symbol>] the events to fire, empty unless the check
|
|
90
|
+
# ran and failed often enough
|
|
91
|
+
#
|
|
92
|
+
# @raise [StandardError] whatever the check raised, other than a
|
|
93
|
+
# System::PsError, after logging it
|
|
94
|
+
#
|
|
95
|
+
# @api private
|
|
96
|
+
def run(pid, tick_number = Integer(Time.now))
|
|
97
|
+
last_ran_at = @last_ran_at
|
|
98
|
+
if last_ran_at.nil? || (last_ran_at + @every) <= tick_number
|
|
25
99
|
@last_ran_at = tick_number
|
|
26
100
|
|
|
27
101
|
begin
|
|
28
102
|
value = @process_condition.run(pid, @include_children)
|
|
29
|
-
rescue => e
|
|
103
|
+
rescue System::PsError => e
|
|
104
|
+
# No data this time, so leave the history alone and check again next interval
|
|
105
|
+
logger.warning("#{@name}: #{e}")
|
|
106
|
+
return EMPTY_ARRAY
|
|
107
|
+
rescue StandardError => e
|
|
30
108
|
logger.err(e.backtrace)
|
|
31
|
-
raise
|
|
109
|
+
raise
|
|
32
110
|
end
|
|
33
111
|
|
|
34
112
|
@history << HistoryValue.new(@process_condition.format_value(value), @process_condition.check(value))
|
|
@@ -39,14 +117,31 @@ module Bluepill
|
|
|
39
117
|
EMPTY_ARRAY
|
|
40
118
|
end
|
|
41
119
|
|
|
120
|
+
# Forget the values the check read, as when the process changes state
|
|
121
|
+
#
|
|
122
|
+
# @return [Util::RotationalArray<HistoryValue>] the new, empty history
|
|
123
|
+
#
|
|
124
|
+
# @api private
|
|
42
125
|
def clear_history!
|
|
43
126
|
@history = Util::RotationalArray.new(@times.last)
|
|
44
127
|
end
|
|
45
128
|
|
|
129
|
+
# Return whether enough of the last values failed for the watch to fire
|
|
130
|
+
#
|
|
131
|
+
# @return [Boolean] true when the failures reach the times the check
|
|
132
|
+
# allows
|
|
133
|
+
#
|
|
134
|
+
# @api private
|
|
46
135
|
def fired?
|
|
47
136
|
@history.count { |v| !v.critical } >= @times.first
|
|
48
137
|
end
|
|
49
138
|
|
|
139
|
+
# Return the check's name and history, as the log shows it
|
|
140
|
+
#
|
|
141
|
+
# @return [String] such as "mem_usage: [312MB, 401MB*]", where * marks a
|
|
142
|
+
# value that failed
|
|
143
|
+
#
|
|
144
|
+
# @api private
|
|
50
145
|
def to_s
|
|
51
146
|
data = @history.collect { |v| "#{v.value}#{'*' unless v.critical}" }.join(', ')
|
|
52
147
|
"#{@name}: [#{data}]\n"
|