bluepill 0.1.3 → 0.1.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/CHANGELOG.md +835 -0
- data/CONTRIBUTING.md +31 -0
- data/LICENSE +1 -1
- data/README.md +66 -29
- data/bin/bluepill +1 -1
- data/bluepill.gemspec +17 -6
- data/lib/bluepill/application.rb +321 -46
- data/lib/bluepill/condition_watch.rb +103 -8
- data/lib/bluepill/controller.rb +124 -13
- data/lib/bluepill/dsl/app_proxy.rb +102 -3
- data/lib/bluepill/dsl/process_factory.rb +183 -60
- data/lib/bluepill/dsl/process_proxy.rb +132 -14
- data/lib/bluepill/dsl.rb +27 -5
- data/lib/bluepill/group.rb +142 -9
- data/lib/bluepill/logger.rb +122 -5
- data/lib/bluepill/process.rb +575 -201
- data/lib/bluepill/process_commands.rb +264 -0
- data/lib/bluepill/process_conditions/always_true.rb +18 -4
- data/lib/bluepill/process_conditions/cpu_usage.rb +34 -0
- data/lib/bluepill/process_conditions/file_time.rb +40 -4
- data/lib/bluepill/process_conditions/http.rb +81 -23
- data/lib/bluepill/process_conditions/mem_usage.rb +61 -2
- data/lib/bluepill/process_conditions/process_condition.rb +43 -0
- data/lib/bluepill/process_conditions/running_time.rb +36 -1
- data/lib/bluepill/process_conditions/zombie_process.rb +26 -5
- data/lib/bluepill/process_conditions.rb +15 -1
- data/lib/bluepill/process_journal.rb +374 -114
- data/lib/bluepill/process_statistics.rb +33 -3
- data/lib/bluepill/socket.rb +63 -16
- data/lib/bluepill/system/commands.rb +286 -0
- data/lib/bluepill/system.rb +223 -228
- data/lib/bluepill/trigger.rb +110 -14
- data/lib/bluepill/triggers/flapping.rb +84 -3
- data/lib/bluepill/util/rotational_array.rb +24 -1
- data/lib/bluepill/version.rb +71 -19
- data/lib/bluepill.rb +25 -2
- data/sig/bluepill.rbs +941 -0
- data/sig/manifest.yaml +7 -0
- metadata +16 -34
- data/bin/sample_forking_server +0 -66
- data/lib/bluepill/application/client.rb +0 -6
- data/lib/bluepill/application/server.rb +0 -21
data/lib/bluepill/process.rb
CHANGED
|
@@ -5,9 +5,20 @@ require 'state_machine'
|
|
|
5
5
|
require 'daemons'
|
|
6
6
|
require 'bluepill/system'
|
|
7
7
|
require 'bluepill/process_journal'
|
|
8
|
+
require 'bluepill/process_commands'
|
|
8
9
|
|
|
9
10
|
module Bluepill
|
|
11
|
+
# A process that bluepill monitors
|
|
12
|
+
#
|
|
13
|
+
# Its state machine moves it between unmonitored, up, down, starting,
|
|
14
|
+
# stopping and restarting. Every tick checks that it's still running,
|
|
15
|
+
# runs its watches, and fires the events they ask for.
|
|
16
|
+
#
|
|
17
|
+
# @api private
|
|
10
18
|
class Process
|
|
19
|
+
include ProcessCommands
|
|
20
|
+
|
|
21
|
+
# The options a config can set for a process
|
|
11
22
|
CONFIGURABLE_ATTRIBUTES = [
|
|
12
23
|
:pre_start_command,
|
|
13
24
|
:start_command,
|
|
@@ -51,9 +62,301 @@ module Bluepill
|
|
|
51
62
|
|
|
52
63
|
].freeze
|
|
53
64
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
65
|
+
# Return the process's name, which commands and logs use
|
|
66
|
+
#
|
|
67
|
+
# @return [String]
|
|
68
|
+
#
|
|
69
|
+
# @api private
|
|
70
|
+
attr_accessor :name
|
|
71
|
+
|
|
72
|
+
# Return the checks that run on the process every interval
|
|
73
|
+
#
|
|
74
|
+
# @return [Array<ConditionWatch>]
|
|
75
|
+
#
|
|
76
|
+
# @api private
|
|
77
|
+
attr_accessor :watches
|
|
78
|
+
|
|
79
|
+
# Return the triggers notified of the process's transitions
|
|
80
|
+
#
|
|
81
|
+
# @return [Array<Trigger>]
|
|
82
|
+
#
|
|
83
|
+
# @api private
|
|
84
|
+
attr_accessor :triggers
|
|
85
|
+
|
|
86
|
+
# Return the time, in epoch seconds, before which ticks are skipped
|
|
87
|
+
#
|
|
88
|
+
# Set while a start, stop or restart command has its grace time to take
|
|
89
|
+
# effect.
|
|
90
|
+
#
|
|
91
|
+
# @return [Integer, nil]
|
|
92
|
+
#
|
|
93
|
+
# @api private
|
|
94
|
+
attr_accessor :skip_ticks_until
|
|
95
|
+
|
|
96
|
+
# Return whether the process was running when last checked this tick
|
|
97
|
+
#
|
|
98
|
+
# tick clears it, so the first check of each tick signals the process.
|
|
99
|
+
#
|
|
100
|
+
# @return [Boolean, nil]
|
|
101
|
+
#
|
|
102
|
+
# @api private
|
|
103
|
+
attr_accessor :process_running
|
|
104
|
+
|
|
105
|
+
# Return the command run before every start, if any
|
|
106
|
+
#
|
|
107
|
+
# Bluepill waits for it to finish, and starts the process even if it exits
|
|
108
|
+
# with a non-zero status. It runs in the process's working_dir, with its
|
|
109
|
+
# environment, uid and gid.
|
|
110
|
+
#
|
|
111
|
+
# @return [String, nil]
|
|
112
|
+
#
|
|
113
|
+
# @api private
|
|
114
|
+
attr_accessor :pre_start_command
|
|
115
|
+
|
|
116
|
+
# Return the command that starts the process
|
|
117
|
+
#
|
|
118
|
+
# Bluepill splits it into words and runs it without a shell.
|
|
119
|
+
#
|
|
120
|
+
# @return [String, nil]
|
|
121
|
+
#
|
|
122
|
+
# @api private
|
|
123
|
+
attr_accessor :start_command
|
|
124
|
+
|
|
125
|
+
# Return the command that stops the process, if any
|
|
126
|
+
#
|
|
127
|
+
# <tt>{{PID}}</tt> in it is replaced with the process's pid. Without
|
|
128
|
+
# one, bluepill sends stop_signals, or TERM.
|
|
129
|
+
#
|
|
130
|
+
# @return [String, nil]
|
|
131
|
+
#
|
|
132
|
+
# @api private
|
|
133
|
+
attr_accessor :stop_command
|
|
134
|
+
|
|
135
|
+
# Return the command that restarts the process, if any
|
|
136
|
+
#
|
|
137
|
+
# <tt>{{PID}}</tt> in it is replaced with the process's pid. Without
|
|
138
|
+
# one, bluepill stops the process and starts it again.
|
|
139
|
+
#
|
|
140
|
+
# @return [String, nil]
|
|
141
|
+
#
|
|
142
|
+
# @api private
|
|
143
|
+
attr_accessor :restart_command
|
|
144
|
+
|
|
145
|
+
# Return the file a daemonized process's standard output goes to
|
|
146
|
+
#
|
|
147
|
+
# @return [String, nil]
|
|
148
|
+
#
|
|
149
|
+
# @api private
|
|
150
|
+
attr_accessor :stdout
|
|
151
|
+
|
|
152
|
+
# Return the file a daemonized process's standard error goes to
|
|
153
|
+
#
|
|
154
|
+
# @return [String, nil]
|
|
155
|
+
#
|
|
156
|
+
# @api private
|
|
157
|
+
attr_accessor :stderr
|
|
158
|
+
|
|
159
|
+
# Return the file a daemonized process's standard input comes from
|
|
160
|
+
#
|
|
161
|
+
# @return [String, nil]
|
|
162
|
+
#
|
|
163
|
+
# @api private
|
|
164
|
+
attr_accessor :stdin
|
|
165
|
+
|
|
166
|
+
# Return whether bluepill daemonizes the process and writes its pid file
|
|
167
|
+
#
|
|
168
|
+
# Set it for a process that doesn't daemonize itself.
|
|
169
|
+
#
|
|
170
|
+
# @return [Boolean, nil]
|
|
171
|
+
#
|
|
172
|
+
# @api private
|
|
173
|
+
attr_accessor :daemonize
|
|
174
|
+
|
|
175
|
+
# Return the file that holds the process's pid
|
|
176
|
+
#
|
|
177
|
+
# @return [String, nil]
|
|
178
|
+
#
|
|
179
|
+
# @api private
|
|
180
|
+
attr_accessor :pid_file
|
|
181
|
+
|
|
182
|
+
# Return the directory the process's commands run in
|
|
183
|
+
#
|
|
184
|
+
# A daemonized process starts in / unless this is set.
|
|
185
|
+
#
|
|
186
|
+
# @return [String, nil]
|
|
187
|
+
#
|
|
188
|
+
# @api private
|
|
189
|
+
attr_accessor :working_dir
|
|
190
|
+
|
|
191
|
+
# Return the environment variables the process's commands get
|
|
192
|
+
#
|
|
193
|
+
# @return [Hash{#to_s => #to_s}, nil]
|
|
194
|
+
#
|
|
195
|
+
# @api private
|
|
196
|
+
attr_accessor :environment
|
|
197
|
+
|
|
198
|
+
# Return how long to wait for the process to start, in seconds
|
|
199
|
+
#
|
|
200
|
+
# Bluepill skips ticks for this long after a start. The default is 3.
|
|
201
|
+
#
|
|
202
|
+
# @return [Integer, Float]
|
|
203
|
+
#
|
|
204
|
+
# @api private
|
|
205
|
+
attr_accessor :start_grace_time
|
|
206
|
+
|
|
207
|
+
# Return how long to wait for the process to stop, in seconds
|
|
208
|
+
#
|
|
209
|
+
# Bluepill skips ticks for this long after a stop. The default is 3.
|
|
210
|
+
#
|
|
211
|
+
# @return [Integer, Float]
|
|
212
|
+
#
|
|
213
|
+
# @api private
|
|
214
|
+
attr_accessor :stop_grace_time
|
|
215
|
+
|
|
216
|
+
# Return how long to wait for the process to restart, in seconds
|
|
217
|
+
#
|
|
218
|
+
# Bluepill skips ticks for this long after a restart. The default is 3.
|
|
219
|
+
#
|
|
220
|
+
# @return [Integer, Float]
|
|
221
|
+
#
|
|
222
|
+
# @api private
|
|
223
|
+
attr_accessor :restart_grace_time
|
|
224
|
+
|
|
225
|
+
# Return the user the process's commands run as
|
|
226
|
+
#
|
|
227
|
+
# @return [String, nil]
|
|
228
|
+
#
|
|
229
|
+
# @api private
|
|
230
|
+
attr_accessor :uid
|
|
231
|
+
|
|
232
|
+
# Return the group the process's commands run as
|
|
233
|
+
#
|
|
234
|
+
# @return [String, nil]
|
|
235
|
+
#
|
|
236
|
+
# @api private
|
|
237
|
+
attr_accessor :gid
|
|
238
|
+
|
|
239
|
+
# Return whether to read the pid file only until it gives a pid
|
|
240
|
+
#
|
|
241
|
+
# True by default. Set it to false for a process whose pid changes, such
|
|
242
|
+
# as one checked for zombies.
|
|
243
|
+
#
|
|
244
|
+
# @return [Boolean]
|
|
245
|
+
#
|
|
246
|
+
# @api private
|
|
247
|
+
attr_accessor :cache_actual_pid
|
|
248
|
+
|
|
249
|
+
# Return whether bluepill also watches the process's children
|
|
250
|
+
#
|
|
251
|
+
# @return [Boolean]
|
|
252
|
+
#
|
|
253
|
+
# @api private
|
|
254
|
+
attr_accessor :monitor_children
|
|
255
|
+
|
|
256
|
+
# Return the factory that makes a Process for each child found
|
|
257
|
+
#
|
|
258
|
+
# monitor_children sets it, with the children's checks.
|
|
259
|
+
#
|
|
260
|
+
# @return [ProcessFactory, nil]
|
|
261
|
+
#
|
|
262
|
+
# @api private
|
|
263
|
+
attr_accessor :child_process_factory
|
|
264
|
+
|
|
265
|
+
# Return the command whose output is the process's pid, if any
|
|
266
|
+
#
|
|
267
|
+
# Used instead of pid_file when it's set.
|
|
268
|
+
#
|
|
269
|
+
# @return [String, nil]
|
|
270
|
+
#
|
|
271
|
+
# @api private
|
|
272
|
+
attr_accessor :pid_command
|
|
273
|
+
|
|
274
|
+
# Return whether bluepill starts the process when it loads
|
|
275
|
+
#
|
|
276
|
+
# Only false keeps it from starting: the process is then unmonitored.
|
|
277
|
+
#
|
|
278
|
+
# @return [Boolean, nil]
|
|
279
|
+
#
|
|
280
|
+
# @api private
|
|
281
|
+
attr_accessor :auto_start
|
|
282
|
+
|
|
283
|
+
# Return the extra groups the process's commands run with
|
|
284
|
+
#
|
|
285
|
+
# @return [Array<String>, nil]
|
|
286
|
+
#
|
|
287
|
+
# @api private
|
|
288
|
+
attr_accessor :supplementary_groups
|
|
289
|
+
|
|
290
|
+
# Return the signals that stop the process, with the delays between
|
|
291
|
+
#
|
|
292
|
+
# A signal, then a delay in seconds and a signal, and so on, such as
|
|
293
|
+
# [:quit, 30, :term, 5, :kill]. Each later signal is only sent if the
|
|
294
|
+
# process is still running.
|
|
295
|
+
#
|
|
296
|
+
# @return [Array<Symbol, Integer>, nil]
|
|
297
|
+
#
|
|
298
|
+
# @api private
|
|
299
|
+
attr_accessor :stop_signals
|
|
300
|
+
|
|
301
|
+
# Return the event to dispatch when the start command takes too long
|
|
302
|
+
#
|
|
303
|
+
# The default is start, which tries again.
|
|
304
|
+
#
|
|
305
|
+
# @return [String, Symbol]
|
|
306
|
+
#
|
|
307
|
+
# @api private
|
|
308
|
+
attr_accessor :on_start_timeout
|
|
309
|
+
|
|
310
|
+
# Return whether a group's start runs without waiting for this process
|
|
311
|
+
#
|
|
312
|
+
# @return [Boolean]
|
|
313
|
+
#
|
|
314
|
+
# @api private
|
|
315
|
+
attr_accessor :group_start_noblock
|
|
316
|
+
|
|
317
|
+
# Return whether a group's restart runs without waiting for this process
|
|
318
|
+
#
|
|
319
|
+
# @return [Boolean]
|
|
320
|
+
#
|
|
321
|
+
# @api private
|
|
322
|
+
attr_accessor :group_restart_noblock
|
|
323
|
+
|
|
324
|
+
# Return whether a group's stop runs without waiting for this process
|
|
325
|
+
#
|
|
326
|
+
# @return [Boolean]
|
|
327
|
+
#
|
|
328
|
+
# @api private
|
|
329
|
+
attr_accessor :group_stop_noblock
|
|
330
|
+
|
|
331
|
+
# Return whether a group's unmonitor runs without waiting for it
|
|
332
|
+
#
|
|
333
|
+
# @return [Boolean]
|
|
334
|
+
#
|
|
335
|
+
# @api private
|
|
336
|
+
attr_accessor :group_unmonitor_noblock
|
|
337
|
+
|
|
338
|
+
# Return the processes made for the process's children
|
|
339
|
+
#
|
|
340
|
+
# Only filled when monitor_children is set.
|
|
341
|
+
#
|
|
342
|
+
# @return [Array<Process>]
|
|
343
|
+
#
|
|
344
|
+
# @api private
|
|
345
|
+
attr_reader :children
|
|
346
|
+
|
|
347
|
+
# Return the process's recent events
|
|
348
|
+
#
|
|
349
|
+
# @return [ProcessStatistics]
|
|
350
|
+
#
|
|
351
|
+
# @api private
|
|
352
|
+
attr_reader :statistics
|
|
353
|
+
|
|
354
|
+
# Return the logger, prefixed with the process's name
|
|
355
|
+
#
|
|
356
|
+
# @return [Logger]
|
|
357
|
+
#
|
|
358
|
+
# @api private
|
|
359
|
+
attr_reader :logger
|
|
57
360
|
|
|
58
361
|
state_machine initial: :unmonitored do
|
|
59
362
|
# These are the idle states, i.e. only an event (either external or internal) will trigger a transition.
|
|
@@ -109,6 +412,21 @@ module Bluepill
|
|
|
109
412
|
after_transition any => any, do: :record_transition
|
|
110
413
|
end
|
|
111
414
|
|
|
415
|
+
# Create a process from its checks and options
|
|
416
|
+
#
|
|
417
|
+
# Each check is a watch, or a trigger if one has its name. Options that
|
|
418
|
+
# aren't given keep their defaults: 3 second grace times, a cached pid,
|
|
419
|
+
# and children left alone.
|
|
420
|
+
#
|
|
421
|
+
# @param [String] process_name the process's name
|
|
422
|
+
# @param [Hash{Symbol => Hash{Symbol => Object}}] checks options by
|
|
423
|
+
# check name
|
|
424
|
+
# @param [Hash{Symbol => Object}] options the process's options, such as
|
|
425
|
+
# CONFIGURABLE_ATTRIBUTES, :logger and :actual_pid
|
|
426
|
+
#
|
|
427
|
+
# @return [void]
|
|
428
|
+
#
|
|
429
|
+
# @api private
|
|
112
430
|
def initialize(process_name, checks, options = {})
|
|
113
431
|
@name = process_name
|
|
114
432
|
@event_mutex = Monitor.new
|
|
@@ -137,15 +455,24 @@ module Bluepill
|
|
|
137
455
|
@group_start_noblock = @group_stop_noblock = @group_restart_noblock = @group_unmonitor_noblock = true
|
|
138
456
|
|
|
139
457
|
CONFIGURABLE_ATTRIBUTES.each do |attribute_name|
|
|
140
|
-
|
|
458
|
+
public_send("#{attribute_name}=", options.fetch(attribute_name)) if options.key?(attribute_name)
|
|
141
459
|
end
|
|
142
460
|
|
|
143
461
|
# Let state_machine do its initialization stuff
|
|
144
462
|
super() # no arguments intentional
|
|
145
463
|
end
|
|
146
464
|
|
|
465
|
+
# Check on the process once a tick
|
|
466
|
+
#
|
|
467
|
+
# Unless ticks are being skipped, fire the state machine's tick event, and
|
|
468
|
+
# then, for a process that's up, run its watches and tick its children.
|
|
469
|
+
#
|
|
470
|
+
# @return [void]
|
|
471
|
+
#
|
|
472
|
+
# @api private
|
|
147
473
|
def tick
|
|
148
474
|
return if skipping_ticks?
|
|
475
|
+
|
|
149
476
|
self.skip_ticks_until = nil
|
|
150
477
|
|
|
151
478
|
# clear the memoization per tick
|
|
@@ -154,33 +481,61 @@ module Bluepill
|
|
|
154
481
|
# Deal with thread cleanup here since the stopping state isn't used
|
|
155
482
|
clean_threads if unmonitored?
|
|
156
483
|
|
|
157
|
-
# run state machine transitions
|
|
158
|
-
|
|
484
|
+
# run state machine transitions. state_machine defines tick in a
|
|
485
|
+
# module it creates, which the signatures can't name.
|
|
486
|
+
super # steep:ignore UnexpectedSuper
|
|
159
487
|
|
|
160
488
|
return unless up?
|
|
489
|
+
|
|
161
490
|
run_watches
|
|
162
491
|
|
|
163
492
|
return unless monitor_children?
|
|
493
|
+
|
|
164
494
|
refresh_children!
|
|
165
495
|
children.each(&:tick)
|
|
166
496
|
end
|
|
167
497
|
|
|
498
|
+
# Set the logger, for the process and for its watches and triggers
|
|
499
|
+
#
|
|
500
|
+
# @param [Logger] logger the logger to use
|
|
501
|
+
#
|
|
502
|
+
# @return [void]
|
|
503
|
+
#
|
|
504
|
+
# @api private
|
|
168
505
|
def logger=(logger)
|
|
169
506
|
@logger = logger
|
|
170
507
|
watches.each { |w| w.logger = logger }
|
|
171
508
|
triggers.each { |t| t.logger = logger }
|
|
172
509
|
end
|
|
173
510
|
|
|
174
|
-
#
|
|
511
|
+
# Record an event and fire it on the state machine
|
|
512
|
+
#
|
|
513
|
+
# @param [Symbol, String] event the event, such as :start or :restart
|
|
514
|
+
# @param [String, nil] reason why, for the process's event history
|
|
515
|
+
#
|
|
516
|
+
# @return [Boolean] whether the state machine made the transition
|
|
517
|
+
#
|
|
518
|
+
# @api private
|
|
175
519
|
def dispatch!(event, reason = nil)
|
|
176
520
|
@event_mutex.synchronize do
|
|
177
521
|
@statistics.record_event(event, reason)
|
|
178
|
-
|
|
522
|
+
public_send(event)
|
|
179
523
|
end
|
|
180
524
|
end
|
|
181
525
|
|
|
526
|
+
# Clear the watches' history after the process changes state
|
|
527
|
+
#
|
|
528
|
+
# A process that's monitoring its children also clears their list, so it
|
|
529
|
+
# finds them again. A loopback, such as up to up, changes nothing.
|
|
530
|
+
#
|
|
531
|
+
# @param [#from_name, #to_name, #loopback?] transition the transition
|
|
532
|
+
#
|
|
533
|
+
# @return [void]
|
|
534
|
+
#
|
|
535
|
+
# @api private
|
|
182
536
|
def record_transition(transition)
|
|
183
537
|
return if transition.loopback?
|
|
538
|
+
|
|
184
539
|
@transitioned = true
|
|
185
540
|
|
|
186
541
|
# When a process changes state, we should clear the memory of all the watches
|
|
@@ -194,56 +549,105 @@ module Bluepill
|
|
|
194
549
|
logger.info "Going from #{transition.from_name} => #{transition.to_name}"
|
|
195
550
|
end
|
|
196
551
|
|
|
552
|
+
# Tell every trigger about a transition
|
|
553
|
+
#
|
|
554
|
+
# @param [#from_name, #to_name, #loopback?] transition the transition
|
|
555
|
+
#
|
|
556
|
+
# @return [void]
|
|
557
|
+
#
|
|
558
|
+
# @raise [StandardError] what a trigger raised, after logging it
|
|
559
|
+
#
|
|
560
|
+
# @api private
|
|
197
561
|
def notify_triggers(transition)
|
|
198
562
|
triggers.each do |trigger|
|
|
199
563
|
begin
|
|
200
564
|
trigger.notify(transition)
|
|
201
|
-
rescue => e
|
|
565
|
+
rescue StandardError => e
|
|
202
566
|
logger.err e.backtrace
|
|
203
|
-
raise
|
|
567
|
+
raise
|
|
204
568
|
end
|
|
205
569
|
end
|
|
206
570
|
end
|
|
207
571
|
|
|
208
|
-
#
|
|
209
|
-
|
|
572
|
+
# Add a check to run on the process every interval
|
|
573
|
+
#
|
|
574
|
+
# @param [Symbol] name the check's name, such as :mem_usage
|
|
575
|
+
# @param [Hash{Symbol => Object}] options the check's options
|
|
576
|
+
#
|
|
577
|
+
# @return [Array<ConditionWatch>] the process's watches
|
|
578
|
+
#
|
|
579
|
+
# @api private
|
|
580
|
+
def add_watch(name, options)
|
|
210
581
|
watches << ConditionWatch.new(name, options.merge(logger: logger))
|
|
211
582
|
end
|
|
212
583
|
|
|
213
|
-
|
|
584
|
+
# Add a trigger to notify of the process's transitions
|
|
585
|
+
#
|
|
586
|
+
# @param [Symbol] name the trigger's name, such as :flapping
|
|
587
|
+
# @param [Hash{Symbol => Object}] options the trigger's options
|
|
588
|
+
#
|
|
589
|
+
# @return [Array<Trigger>] the process's triggers
|
|
590
|
+
#
|
|
591
|
+
# @api private
|
|
592
|
+
def add_trigger(name, options)
|
|
214
593
|
triggers << Trigger[name].new(self, options.merge(logger: logger))
|
|
215
594
|
end
|
|
216
595
|
|
|
596
|
+
# Run every watch, and dispatch the events they fire
|
|
597
|
+
#
|
|
598
|
+
# Each watch runs in a thread of its own. Once an event changes the
|
|
599
|
+
# process's state, the events after it are dropped.
|
|
600
|
+
#
|
|
601
|
+
# @return [void]
|
|
602
|
+
#
|
|
603
|
+
# @api private
|
|
217
604
|
def run_watches
|
|
218
|
-
now = Time.now
|
|
605
|
+
now = Integer(Time.now)
|
|
219
606
|
|
|
220
607
|
threads = watches.collect do |watch|
|
|
221
|
-
[watch, Thread.new {
|
|
608
|
+
[watch, Thread.new { watch.run(actual_pid, now) }] #: [ConditionWatch, Thread]
|
|
222
609
|
end
|
|
223
610
|
|
|
224
611
|
@transitioned = false
|
|
225
612
|
|
|
226
613
|
threads.each_with_object([]) do |(watch, thread), events|
|
|
227
|
-
thread.
|
|
228
|
-
next if
|
|
229
|
-
|
|
230
|
-
|
|
614
|
+
watch_events = thread.value #: Array[Symbol]
|
|
615
|
+
next if watch_events.empty?
|
|
616
|
+
|
|
617
|
+
logger.info "#{watch.name} dispatched: #{watch_events.join(',')}"
|
|
618
|
+
watch_events.each do |event|
|
|
231
619
|
events << [event, watch.to_s]
|
|
232
620
|
end
|
|
233
621
|
end.each do |event, reason| # rubocop:disable Style/MultilineBlockChain
|
|
234
622
|
break if @transitioned
|
|
623
|
+
|
|
235
624
|
dispatch!(event, reason)
|
|
236
625
|
end
|
|
237
626
|
end
|
|
238
627
|
|
|
628
|
+
# Set the state to up, down or unmonitored when bluepill loads
|
|
629
|
+
#
|
|
630
|
+
# A running process is up. Otherwise it's down, so that it starts, unless
|
|
631
|
+
# auto_start is false.
|
|
632
|
+
#
|
|
633
|
+
# @return [void]
|
|
634
|
+
#
|
|
635
|
+
# @api private
|
|
239
636
|
def determine_initial_state
|
|
240
637
|
if process_running?(true)
|
|
241
638
|
self.state = 'up'
|
|
242
639
|
return
|
|
243
640
|
end
|
|
244
|
-
self.state = (
|
|
641
|
+
self.state = auto_start.equal?(false) ? 'unmonitored' : 'down' # we need to check for false value
|
|
245
642
|
end
|
|
246
643
|
|
|
644
|
+
# Carry out a command from the command line
|
|
645
|
+
#
|
|
646
|
+
# @param [String] cmd start, stop, restart or unmonitor
|
|
647
|
+
#
|
|
648
|
+
# @return [void]
|
|
649
|
+
#
|
|
650
|
+
# @api private
|
|
247
651
|
def handle_user_command(cmd)
|
|
248
652
|
case cmd
|
|
249
653
|
when 'start'
|
|
@@ -256,7 +660,10 @@ module Bluepill
|
|
|
256
660
|
stop_process
|
|
257
661
|
dispatch!(:unmonitor, 'user initiated')
|
|
258
662
|
when 'restart'
|
|
259
|
-
|
|
663
|
+
# Through the state machine, like a check's restart, so the process
|
|
664
|
+
# shows as restarting until it's up again. A state restart can't
|
|
665
|
+
# leave, such as unmonitored, restarts it directly.
|
|
666
|
+
restart_process unless dispatch!(:restart, 'user initiated')
|
|
260
667
|
when 'unmonitor'
|
|
261
668
|
# When the user issues an unmonitor cmd, reset any triggers so that
|
|
262
669
|
# scheduled events gets cleared
|
|
@@ -265,209 +672,205 @@ module Bluepill
|
|
|
265
672
|
end
|
|
266
673
|
end
|
|
267
674
|
|
|
268
|
-
#
|
|
675
|
+
# Return whether the process is running, and forget its pid if not
|
|
676
|
+
#
|
|
677
|
+
# The answer is kept for the rest of the tick.
|
|
678
|
+
#
|
|
679
|
+
# @param [Boolean] force whether to check again this tick
|
|
680
|
+
#
|
|
681
|
+
# @return [Boolean]
|
|
682
|
+
#
|
|
683
|
+
# @api private
|
|
269
684
|
def process_running?(force = false)
|
|
270
685
|
@process_running = nil if force # clear existing state if forced
|
|
271
686
|
|
|
272
687
|
@process_running ||= signal_process(0)
|
|
273
688
|
# the process isn't running, so we should clear the PID
|
|
274
689
|
clear_pid unless @process_running
|
|
275
|
-
@process_running
|
|
276
|
-
end
|
|
277
|
-
|
|
278
|
-
def start_process
|
|
279
|
-
ProcessJournal.kill_all_from_journal(name) # be sure nothing else is running from previous runs
|
|
280
|
-
pre_start_process
|
|
281
|
-
logger.warning "Executing start command: #{start_command}"
|
|
282
|
-
if daemonize?
|
|
283
|
-
daemon_id = System.daemonize(start_command, system_command_options)
|
|
284
|
-
if daemon_id
|
|
285
|
-
ProcessJournal.append_pid_to_journal(name, daemon_id)
|
|
286
|
-
children.each do |child|
|
|
287
|
-
ProcessJournal.append_pid_to_journal(name, child.actual_pid)
|
|
288
|
-
end if monitor_children?
|
|
289
|
-
end
|
|
290
|
-
daemon_id
|
|
291
|
-
else
|
|
292
|
-
# This is a self-daemonizing process
|
|
293
|
-
with_timeout(start_grace_time, on_start_timeout) do
|
|
294
|
-
result = System.execute_blocking(start_command, system_command_options)
|
|
295
|
-
|
|
296
|
-
unless result[:exit_code].zero?
|
|
297
|
-
logger.warning 'Start command execution returned non-zero exit code:'
|
|
298
|
-
logger.warning result.inspect
|
|
299
|
-
end
|
|
300
|
-
end
|
|
301
|
-
end
|
|
302
|
-
|
|
303
|
-
skip_ticks_for(start_grace_time)
|
|
304
|
-
end
|
|
305
|
-
|
|
306
|
-
def pre_start_process
|
|
307
|
-
return unless pre_start_command
|
|
308
|
-
logger.warning "Executing pre start command: #{pre_start_command}"
|
|
309
|
-
result = System.execute_blocking(pre_start_command, system_command_options)
|
|
310
|
-
return if result[:exit_code].zero?
|
|
311
|
-
logger.warning 'Pre start command execution returned non-zero exit code:'
|
|
312
|
-
logger.warning result.inspect
|
|
313
|
-
end
|
|
314
|
-
|
|
315
|
-
def stop_process
|
|
316
|
-
if monitor_children
|
|
317
|
-
System.get_children(actual_pid).each do |child_pid|
|
|
318
|
-
ProcessJournal.append_pid_to_journal(name, child_pid)
|
|
319
|
-
end
|
|
320
|
-
end
|
|
321
|
-
|
|
322
|
-
if stop_command
|
|
323
|
-
cmd = prepare_command(stop_command)
|
|
324
|
-
logger.warning "Executing stop command: #{cmd}"
|
|
325
|
-
|
|
326
|
-
with_timeout(stop_grace_time, 'stop') do
|
|
327
|
-
result = System.execute_blocking(cmd, system_command_options)
|
|
328
|
-
|
|
329
|
-
unless result[:exit_code].zero?
|
|
330
|
-
logger.warning 'Stop command execution returned non-zero exit code:'
|
|
331
|
-
logger.warning result.inspect
|
|
332
|
-
end
|
|
333
|
-
end
|
|
334
|
-
cleanup_process
|
|
335
|
-
elsif stop_signals
|
|
336
|
-
# issue stop signals with configurable delay between each
|
|
337
|
-
logger.warning "Sending stop signals to #{actual_pid}"
|
|
338
|
-
@threads << Thread.new(self, stop_signals.clone) do |process, stop_signals|
|
|
339
|
-
signal = stop_signals.shift
|
|
340
|
-
logger.info "Sending signal #{signal} to #{process.actual_pid}"
|
|
341
|
-
process.signal_process(signal) # send first signal
|
|
342
|
-
|
|
343
|
-
until stop_signals.empty?
|
|
344
|
-
# we already checked to make sure stop_signals had an odd number of items
|
|
345
|
-
delay = stop_signals.shift
|
|
346
|
-
signal = stop_signals.shift
|
|
347
|
-
|
|
348
|
-
logger.debug "Sleeping for #{delay} seconds"
|
|
349
|
-
sleep delay
|
|
350
|
-
# break unless signal_process(0) #break unless the process can be reached
|
|
351
|
-
unless process.signal_process(0)
|
|
352
|
-
logger.debug 'Process has terminated.'
|
|
353
|
-
break
|
|
354
|
-
end
|
|
355
|
-
logger.info "Sending signal #{signal} to #{process.actual_pid}"
|
|
356
|
-
process.signal_process(signal)
|
|
357
|
-
end
|
|
358
|
-
cleanup_process
|
|
359
|
-
end
|
|
360
|
-
else
|
|
361
|
-
logger.warning "Executing default stop command. Sending TERM signal to #{actual_pid}"
|
|
362
|
-
signal_process('TERM')
|
|
363
|
-
cleanup_process
|
|
364
|
-
end
|
|
365
|
-
|
|
366
|
-
skip_ticks_for(stop_grace_time)
|
|
367
|
-
end
|
|
368
|
-
|
|
369
|
-
def restart_process
|
|
370
|
-
if restart_command
|
|
371
|
-
cmd = prepare_command(restart_command)
|
|
372
|
-
|
|
373
|
-
logger.warning "Executing restart command: #{cmd}"
|
|
374
|
-
|
|
375
|
-
with_timeout(restart_grace_time, 'restart') do
|
|
376
|
-
result = System.execute_blocking(cmd, system_command_options)
|
|
377
|
-
|
|
378
|
-
unless result[:exit_code].zero?
|
|
379
|
-
logger.warning 'Restart command execution returned non-zero exit code:'
|
|
380
|
-
logger.warning result.inspect
|
|
381
|
-
end
|
|
382
|
-
end
|
|
383
|
-
|
|
384
|
-
skip_ticks_for(restart_grace_time)
|
|
385
|
-
else
|
|
386
|
-
logger.warning 'No restart_command specified. Must stop and start to restart'
|
|
387
|
-
stop_process
|
|
388
|
-
start_process
|
|
389
|
-
end
|
|
390
|
-
end
|
|
391
|
-
|
|
392
|
-
def cleanup_process
|
|
393
|
-
ProcessJournal.kill_all_from_journal(name) # finish cleanup
|
|
394
|
-
unlink_pid # TODO: we only write the pid file if we daemonize, should we only unlink it if we daemonize?
|
|
690
|
+
@process_running #: bool
|
|
395
691
|
end
|
|
396
692
|
|
|
693
|
+
# Kill the threads the process's commands left running
|
|
694
|
+
#
|
|
695
|
+
# @return [void]
|
|
696
|
+
#
|
|
697
|
+
# @api private
|
|
397
698
|
def clean_threads
|
|
398
699
|
@threads.each(&:kill)
|
|
399
700
|
@threads.clear
|
|
400
701
|
end
|
|
401
702
|
|
|
703
|
+
# Return whether bluepill daemonizes the process
|
|
704
|
+
#
|
|
705
|
+
# @return [Boolean]
|
|
706
|
+
#
|
|
707
|
+
# @api private
|
|
402
708
|
def daemonize?
|
|
403
709
|
!!daemonize
|
|
404
710
|
end
|
|
405
711
|
|
|
712
|
+
# Return whether bluepill watches the process's children
|
|
713
|
+
#
|
|
714
|
+
# @return [Boolean]
|
|
715
|
+
#
|
|
716
|
+
# @api private
|
|
406
717
|
def monitor_children?
|
|
407
718
|
!!monitor_children
|
|
408
719
|
end
|
|
409
720
|
|
|
721
|
+
# Send a signal to the process
|
|
722
|
+
#
|
|
723
|
+
# A process without a pid isn't running, so it gets no signal and nothing
|
|
724
|
+
# is logged. A failure is logged.
|
|
725
|
+
#
|
|
726
|
+
# @param [Integer, String, Symbol] code the signal, such as 0 or :term
|
|
727
|
+
#
|
|
728
|
+
# @return [Boolean] whether the signal was sent
|
|
729
|
+
#
|
|
730
|
+
# @api private
|
|
410
731
|
def signal_process(code)
|
|
411
|
-
code = code.to_s.upcase if code.is_a?(String) || code.
|
|
412
|
-
|
|
732
|
+
code = code.to_s.upcase if code.is_a?(String) || code.instance_of?(Symbol)
|
|
733
|
+
pid = actual_pid
|
|
734
|
+
# Without a pid the process isn't running. That's expected before the
|
|
735
|
+
# first start, and actual_pid already warns about a missing pid file.
|
|
736
|
+
return false unless pid
|
|
737
|
+
|
|
738
|
+
# kill with a pid of 0 or less signals a whole process group, which
|
|
739
|
+
# could include bluepill itself, so only signal a real pid
|
|
740
|
+
raise "invalid pid #{pid}" unless pid > 0
|
|
741
|
+
|
|
742
|
+
::Process.kill(code, pid)
|
|
413
743
|
true
|
|
414
|
-
rescue => e
|
|
415
|
-
logger.err "Failed to signal process #{
|
|
744
|
+
rescue StandardError => e
|
|
745
|
+
logger.err "Failed to signal process #{pid} with code #{code}: #{e}"
|
|
416
746
|
false
|
|
417
747
|
end
|
|
418
748
|
|
|
749
|
+
# Return whether the pid is kept once it's read
|
|
750
|
+
#
|
|
751
|
+
# @return [Boolean]
|
|
752
|
+
#
|
|
753
|
+
# @api private
|
|
419
754
|
def cache_actual_pid?
|
|
420
755
|
!!@cache_actual_pid
|
|
421
756
|
end
|
|
422
757
|
|
|
758
|
+
# Return the process's pid, from pid_command or the pid file
|
|
759
|
+
#
|
|
760
|
+
# @return [Integer, nil] the pid, or nil if there's none yet
|
|
761
|
+
#
|
|
762
|
+
# @api private
|
|
423
763
|
def actual_pid
|
|
424
764
|
pid_command ? pid_from_command : pid_from_file
|
|
425
765
|
end
|
|
426
766
|
|
|
767
|
+
# Return the pid in the pid file, cached unless cache_actual_pid is false
|
|
768
|
+
#
|
|
769
|
+
# A missing pid file is logged as a warning.
|
|
770
|
+
#
|
|
771
|
+
# @return [Integer, nil] the pid, or nil without a pid file or a pid
|
|
772
|
+
#
|
|
773
|
+
# @api private
|
|
427
774
|
def pid_from_file
|
|
428
775
|
return @actual_pid if cache_actual_pid? && @actual_pid
|
|
429
|
-
@actual_pid = begin
|
|
430
|
-
if pid_file
|
|
431
|
-
if File.exist?(pid_file)
|
|
432
|
-
str = File.read(pid_file)
|
|
433
|
-
str.to_i unless str.empty?
|
|
434
|
-
else
|
|
435
|
-
logger.warning("pid_file #{pid_file} does not exist or cannot be read")
|
|
436
|
-
nil
|
|
437
|
-
end
|
|
438
|
-
end
|
|
439
|
-
end
|
|
440
|
-
end
|
|
441
776
|
|
|
777
|
+
@actual_pid = if pid_file
|
|
778
|
+
if File.exist?(pid_file)
|
|
779
|
+
positive_pid(File.read(pid_file).to_i)
|
|
780
|
+
else
|
|
781
|
+
logger.warning("pid_file #{pid_file} does not exist or cannot be read")
|
|
782
|
+
nil
|
|
783
|
+
end
|
|
784
|
+
end
|
|
785
|
+
end
|
|
786
|
+
|
|
787
|
+
# Return the pid that pid_command prints
|
|
788
|
+
#
|
|
789
|
+
# Output that isn't only digits gives no pid.
|
|
790
|
+
#
|
|
791
|
+
# @return [Integer, nil] the pid, or nil if the output isn't one
|
|
792
|
+
#
|
|
793
|
+
# @api private
|
|
442
794
|
def pid_from_command
|
|
443
|
-
|
|
444
|
-
(
|
|
445
|
-
end
|
|
446
|
-
|
|
795
|
+
# Output that is not just digits reads as nil, so as 0
|
|
796
|
+
positive_pid(`#{pid_command}`.strip[/\A\d+\z/].to_i)
|
|
797
|
+
end
|
|
798
|
+
|
|
799
|
+
# Return the pid, unless it's 0 or less
|
|
800
|
+
#
|
|
801
|
+
# An empty or garbled pid file reads as 0, which is never a real process.
|
|
802
|
+
#
|
|
803
|
+
# @param [Integer] pid the pid that was read
|
|
804
|
+
#
|
|
805
|
+
# @return [Integer, nil] the pid, or nil if it's not a real one
|
|
806
|
+
#
|
|
807
|
+
# @api private
|
|
808
|
+
def positive_pid(pid)
|
|
809
|
+
pid if pid > 0
|
|
810
|
+
end
|
|
811
|
+
|
|
812
|
+
# Set the pid, and add it to the process's journal
|
|
813
|
+
#
|
|
814
|
+
# @param [Integer] pid the process's pid
|
|
815
|
+
#
|
|
816
|
+
# @return [void]
|
|
817
|
+
#
|
|
818
|
+
# @api private
|
|
447
819
|
def actual_pid=(pid)
|
|
448
820
|
ProcessJournal.append_pid_to_journal(name, pid) # be sure to always log the pid
|
|
449
821
|
@actual_pid = pid
|
|
450
822
|
end
|
|
451
823
|
|
|
824
|
+
# Forget the process's pid
|
|
825
|
+
#
|
|
826
|
+
# @return [nil]
|
|
827
|
+
#
|
|
828
|
+
# @api private
|
|
452
829
|
def clear_pid
|
|
453
830
|
@actual_pid = nil
|
|
454
831
|
end
|
|
455
832
|
|
|
833
|
+
# Delete the pid file, if there is one
|
|
834
|
+
#
|
|
835
|
+
# @return [void]
|
|
836
|
+
#
|
|
837
|
+
# @api private
|
|
456
838
|
def unlink_pid
|
|
457
839
|
System.delete_if_exists(pid_file)
|
|
458
840
|
end
|
|
459
841
|
|
|
460
|
-
#
|
|
842
|
+
# Skip ticks for a while, such as for a command's grace time
|
|
843
|
+
#
|
|
844
|
+
# The times add up, so a stop followed by a start, which is how
|
|
845
|
+
# restart_process works without a restart_command, waits out both grace
|
|
846
|
+
# times.
|
|
847
|
+
#
|
|
848
|
+
# @param [Integer, Float] seconds how long to skip ticks for
|
|
849
|
+
#
|
|
850
|
+
# @return [Integer] when ticks resume, in epoch seconds
|
|
851
|
+
#
|
|
852
|
+
# @api private
|
|
461
853
|
def skip_ticks_for(seconds)
|
|
462
|
-
|
|
463
|
-
# i.e. if two calls for skip_ticks_for come in for 5 and 10, should it skip for 10 or 15?
|
|
464
|
-
self.skip_ticks_until = (skip_ticks_until || Time.now.to_i) + seconds.to_i
|
|
854
|
+
self.skip_ticks_until = (skip_ticks_until || Integer(Time.now)) + Integer(seconds)
|
|
465
855
|
end
|
|
466
856
|
|
|
857
|
+
# Return whether ticks are being skipped
|
|
858
|
+
#
|
|
859
|
+
# @return [Boolean, nil]
|
|
860
|
+
#
|
|
861
|
+
# @api private
|
|
467
862
|
def skipping_ticks?
|
|
468
|
-
skip_ticks_until && skip_ticks_until > Time.now
|
|
863
|
+
skip_ticks_until && skip_ticks_until > Integer(Time.now)
|
|
469
864
|
end
|
|
470
865
|
|
|
866
|
+
# Update the list of children from the tick's ps table
|
|
867
|
+
#
|
|
868
|
+
# Children that are no longer running are dropped, and a Process is made
|
|
869
|
+
# for each new one, with the checks monitor_children gave.
|
|
870
|
+
#
|
|
871
|
+
# @return [void]
|
|
872
|
+
#
|
|
873
|
+
# @api private
|
|
471
874
|
def refresh_children!
|
|
472
875
|
# First prune the list of dead children
|
|
473
876
|
@children.delete_if { |child| !child.process_running?(true) }
|
|
@@ -475,48 +878,19 @@ module Bluepill
|
|
|
475
878
|
# Add new found children to the list
|
|
476
879
|
new_children_pids = System.get_children(actual_pid) - @children.collect(&:actual_pid)
|
|
477
880
|
|
|
478
|
-
unless new_children_pids.empty?
|
|
479
|
-
logger.info "Existing children: #{@children.collect(&:actual_pid).join(',')}. Got new children: #{new_children_pids.inspect} for #{actual_pid}"
|
|
480
|
-
end
|
|
881
|
+
logger.info "Existing children: #{@children.collect(&:actual_pid).join(',')}. Got new children: #{new_children_pids} for #{actual_pid}" unless new_children_pids.empty?
|
|
481
882
|
|
|
482
883
|
# Construct a new process wrapper for each new found children
|
|
884
|
+
# monitor_children sets the factory
|
|
885
|
+
factory = child_process_factory #: ProcessFactory
|
|
483
886
|
new_children_pids.each do |child_pid|
|
|
484
887
|
ProcessJournal.append_pid_to_journal(name, child_pid) if daemonize?
|
|
485
888
|
child_name = "<child(pid:#{child_pid})>"
|
|
486
|
-
|
|
889
|
+
child_logger = logger.prefix_with(child_name)
|
|
487
890
|
|
|
488
|
-
child =
|
|
891
|
+
child = factory.create_child_process(child_name, child_pid, child_logger)
|
|
489
892
|
@children << child
|
|
490
893
|
end
|
|
491
894
|
end
|
|
492
|
-
|
|
493
|
-
def prepare_command(command)
|
|
494
|
-
command.to_s.gsub('{{PID}}', actual_pid.to_s)
|
|
495
|
-
end
|
|
496
|
-
|
|
497
|
-
def system_command_options
|
|
498
|
-
{
|
|
499
|
-
uid: uid,
|
|
500
|
-
gid: gid,
|
|
501
|
-
working_dir: working_dir,
|
|
502
|
-
environment: environment,
|
|
503
|
-
pid_file: pid_file,
|
|
504
|
-
logger: logger,
|
|
505
|
-
stdin: stdin,
|
|
506
|
-
stdout: stdout,
|
|
507
|
-
stderr: stderr,
|
|
508
|
-
supplementary_groups: supplementary_groups,
|
|
509
|
-
}
|
|
510
|
-
end
|
|
511
|
-
|
|
512
|
-
def with_timeout(secs, next_state = nil, &blk)
|
|
513
|
-
# Attempt to execute the passed block. If the block takes
|
|
514
|
-
# too long, transition to the indicated next state.
|
|
515
|
-
Timeout.timeout(secs.to_f, &blk)
|
|
516
|
-
rescue Timeout::Error
|
|
517
|
-
logger.err 'Execution is taking longer than expected.'
|
|
518
|
-
logger.err 'Did you forget to tell bluepill to daemonize this process?'
|
|
519
|
-
dispatch!(next_state)
|
|
520
|
-
end
|
|
521
895
|
end
|
|
522
896
|
end
|