bluepill 0.1.3 → 0.1.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +5 -5
- data/CHANGELOG.md +820 -0
- data/CONTRIBUTING.md +31 -0
- data/LICENSE +1 -1
- data/README.md +64 -29
- data/bin/bluepill +1 -1
- data/bluepill.gemspec +17 -6
- data/lib/bluepill/application.rb +309 -41
- data/lib/bluepill/condition_watch.rb +103 -8
- data/lib/bluepill/controller.rb +124 -13
- data/lib/bluepill/dsl/app_proxy.rb +102 -3
- data/lib/bluepill/dsl/process_factory.rb +183 -60
- data/lib/bluepill/dsl/process_proxy.rb +132 -14
- data/lib/bluepill/dsl.rb +27 -5
- data/lib/bluepill/group.rb +142 -9
- data/lib/bluepill/logger.rb +120 -5
- data/lib/bluepill/process.rb +571 -200
- data/lib/bluepill/process_commands.rb +262 -0
- data/lib/bluepill/process_conditions/always_true.rb +18 -4
- data/lib/bluepill/process_conditions/cpu_usage.rb +34 -0
- data/lib/bluepill/process_conditions/file_time.rb +40 -4
- data/lib/bluepill/process_conditions/http.rb +81 -23
- data/lib/bluepill/process_conditions/mem_usage.rb +61 -2
- data/lib/bluepill/process_conditions/process_condition.rb +43 -0
- data/lib/bluepill/process_conditions/running_time.rb +36 -1
- data/lib/bluepill/process_conditions/zombie_process.rb +26 -5
- data/lib/bluepill/process_conditions.rb +15 -1
- data/lib/bluepill/process_journal.rb +256 -104
- data/lib/bluepill/process_statistics.rb +33 -3
- data/lib/bluepill/socket.rb +63 -16
- data/lib/bluepill/system/commands.rb +286 -0
- data/lib/bluepill/system.rb +192 -228
- data/lib/bluepill/trigger.rb +110 -14
- data/lib/bluepill/triggers/flapping.rb +84 -3
- data/lib/bluepill/util/rotational_array.rb +24 -1
- data/lib/bluepill/version.rb +71 -19
- data/lib/bluepill.rb +25 -2
- data/sig/bluepill.rbs +926 -0
- data/sig/manifest.yaml +7 -0
- metadata +16 -34
- data/bin/sample_forking_server +0 -66
- data/lib/bluepill/application/client.rb +0 -6
- data/lib/bluepill/application/server.rb +0 -21
data/lib/bluepill/process.rb
CHANGED
|
@@ -5,9 +5,20 @@ require 'state_machine'
|
|
|
5
5
|
require 'daemons'
|
|
6
6
|
require 'bluepill/system'
|
|
7
7
|
require 'bluepill/process_journal'
|
|
8
|
+
require 'bluepill/process_commands'
|
|
8
9
|
|
|
9
10
|
module Bluepill
|
|
11
|
+
# A process that bluepill monitors
|
|
12
|
+
#
|
|
13
|
+
# Its state machine moves it between unmonitored, up, down, starting,
|
|
14
|
+
# stopping and restarting. Every tick checks that it's still running,
|
|
15
|
+
# runs its watches, and fires the events they ask for.
|
|
16
|
+
#
|
|
17
|
+
# @api private
|
|
10
18
|
class Process
|
|
19
|
+
include ProcessCommands
|
|
20
|
+
|
|
21
|
+
# The options a config can set for a process
|
|
11
22
|
CONFIGURABLE_ATTRIBUTES = [
|
|
12
23
|
:pre_start_command,
|
|
13
24
|
:start_command,
|
|
@@ -51,9 +62,301 @@ module Bluepill
|
|
|
51
62
|
|
|
52
63
|
].freeze
|
|
53
64
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
65
|
+
# Return the process's name, which commands and logs use
|
|
66
|
+
#
|
|
67
|
+
# @return [String]
|
|
68
|
+
#
|
|
69
|
+
# @api private
|
|
70
|
+
attr_accessor :name
|
|
71
|
+
|
|
72
|
+
# Return the checks that run on the process every interval
|
|
73
|
+
#
|
|
74
|
+
# @return [Array<ConditionWatch>]
|
|
75
|
+
#
|
|
76
|
+
# @api private
|
|
77
|
+
attr_accessor :watches
|
|
78
|
+
|
|
79
|
+
# Return the triggers notified of the process's transitions
|
|
80
|
+
#
|
|
81
|
+
# @return [Array<Trigger>]
|
|
82
|
+
#
|
|
83
|
+
# @api private
|
|
84
|
+
attr_accessor :triggers
|
|
85
|
+
|
|
86
|
+
# Return the time, in epoch seconds, before which ticks are skipped
|
|
87
|
+
#
|
|
88
|
+
# Set while a start, stop or restart command has its grace time to take
|
|
89
|
+
# effect.
|
|
90
|
+
#
|
|
91
|
+
# @return [Integer, nil]
|
|
92
|
+
#
|
|
93
|
+
# @api private
|
|
94
|
+
attr_accessor :skip_ticks_until
|
|
95
|
+
|
|
96
|
+
# Return whether the process was running when last checked this tick
|
|
97
|
+
#
|
|
98
|
+
# tick clears it, so the first check of each tick signals the process.
|
|
99
|
+
#
|
|
100
|
+
# @return [Boolean, nil]
|
|
101
|
+
#
|
|
102
|
+
# @api private
|
|
103
|
+
attr_accessor :process_running
|
|
104
|
+
|
|
105
|
+
# Return the command run before every start, if any
|
|
106
|
+
#
|
|
107
|
+
# Bluepill waits for it to finish, and starts the process even if it exits
|
|
108
|
+
# with a non-zero status. It runs in the process's working_dir, with its
|
|
109
|
+
# environment, uid and gid.
|
|
110
|
+
#
|
|
111
|
+
# @return [String, nil]
|
|
112
|
+
#
|
|
113
|
+
# @api private
|
|
114
|
+
attr_accessor :pre_start_command
|
|
115
|
+
|
|
116
|
+
# Return the command that starts the process
|
|
117
|
+
#
|
|
118
|
+
# Bluepill splits it into words and runs it without a shell.
|
|
119
|
+
#
|
|
120
|
+
# @return [String, nil]
|
|
121
|
+
#
|
|
122
|
+
# @api private
|
|
123
|
+
attr_accessor :start_command
|
|
124
|
+
|
|
125
|
+
# Return the command that stops the process, if any
|
|
126
|
+
#
|
|
127
|
+
# <tt>{{PID}}</tt> in it is replaced with the process's pid. Without
|
|
128
|
+
# one, bluepill sends stop_signals, or TERM.
|
|
129
|
+
#
|
|
130
|
+
# @return [String, nil]
|
|
131
|
+
#
|
|
132
|
+
# @api private
|
|
133
|
+
attr_accessor :stop_command
|
|
134
|
+
|
|
135
|
+
# Return the command that restarts the process, if any
|
|
136
|
+
#
|
|
137
|
+
# <tt>{{PID}}</tt> in it is replaced with the process's pid. Without
|
|
138
|
+
# one, bluepill stops the process and starts it again.
|
|
139
|
+
#
|
|
140
|
+
# @return [String, nil]
|
|
141
|
+
#
|
|
142
|
+
# @api private
|
|
143
|
+
attr_accessor :restart_command
|
|
144
|
+
|
|
145
|
+
# Return the file a daemonized process's standard output goes to
|
|
146
|
+
#
|
|
147
|
+
# @return [String, nil]
|
|
148
|
+
#
|
|
149
|
+
# @api private
|
|
150
|
+
attr_accessor :stdout
|
|
151
|
+
|
|
152
|
+
# Return the file a daemonized process's standard error goes to
|
|
153
|
+
#
|
|
154
|
+
# @return [String, nil]
|
|
155
|
+
#
|
|
156
|
+
# @api private
|
|
157
|
+
attr_accessor :stderr
|
|
158
|
+
|
|
159
|
+
# Return the file a daemonized process's standard input comes from
|
|
160
|
+
#
|
|
161
|
+
# @return [String, nil]
|
|
162
|
+
#
|
|
163
|
+
# @api private
|
|
164
|
+
attr_accessor :stdin
|
|
165
|
+
|
|
166
|
+
# Return whether bluepill daemonizes the process and writes its pid file
|
|
167
|
+
#
|
|
168
|
+
# Set it for a process that doesn't daemonize itself.
|
|
169
|
+
#
|
|
170
|
+
# @return [Boolean, nil]
|
|
171
|
+
#
|
|
172
|
+
# @api private
|
|
173
|
+
attr_accessor :daemonize
|
|
174
|
+
|
|
175
|
+
# Return the file that holds the process's pid
|
|
176
|
+
#
|
|
177
|
+
# @return [String, nil]
|
|
178
|
+
#
|
|
179
|
+
# @api private
|
|
180
|
+
attr_accessor :pid_file
|
|
181
|
+
|
|
182
|
+
# Return the directory the process's commands run in
|
|
183
|
+
#
|
|
184
|
+
# A daemonized process starts in / unless this is set.
|
|
185
|
+
#
|
|
186
|
+
# @return [String, nil]
|
|
187
|
+
#
|
|
188
|
+
# @api private
|
|
189
|
+
attr_accessor :working_dir
|
|
190
|
+
|
|
191
|
+
# Return the environment variables the process's commands get
|
|
192
|
+
#
|
|
193
|
+
# @return [Hash{#to_s => #to_s}, nil]
|
|
194
|
+
#
|
|
195
|
+
# @api private
|
|
196
|
+
attr_accessor :environment
|
|
197
|
+
|
|
198
|
+
# Return how long to wait for the process to start, in seconds
|
|
199
|
+
#
|
|
200
|
+
# Bluepill skips ticks for this long after a start. The default is 3.
|
|
201
|
+
#
|
|
202
|
+
# @return [Integer, Float]
|
|
203
|
+
#
|
|
204
|
+
# @api private
|
|
205
|
+
attr_accessor :start_grace_time
|
|
206
|
+
|
|
207
|
+
# Return how long to wait for the process to stop, in seconds
|
|
208
|
+
#
|
|
209
|
+
# Bluepill skips ticks for this long after a stop. The default is 3.
|
|
210
|
+
#
|
|
211
|
+
# @return [Integer, Float]
|
|
212
|
+
#
|
|
213
|
+
# @api private
|
|
214
|
+
attr_accessor :stop_grace_time
|
|
215
|
+
|
|
216
|
+
# Return how long to wait for the process to restart, in seconds
|
|
217
|
+
#
|
|
218
|
+
# Bluepill skips ticks for this long after a restart. The default is 3.
|
|
219
|
+
#
|
|
220
|
+
# @return [Integer, Float]
|
|
221
|
+
#
|
|
222
|
+
# @api private
|
|
223
|
+
attr_accessor :restart_grace_time
|
|
224
|
+
|
|
225
|
+
# Return the user the process's commands run as
|
|
226
|
+
#
|
|
227
|
+
# @return [String, nil]
|
|
228
|
+
#
|
|
229
|
+
# @api private
|
|
230
|
+
attr_accessor :uid
|
|
231
|
+
|
|
232
|
+
# Return the group the process's commands run as
|
|
233
|
+
#
|
|
234
|
+
# @return [String, nil]
|
|
235
|
+
#
|
|
236
|
+
# @api private
|
|
237
|
+
attr_accessor :gid
|
|
238
|
+
|
|
239
|
+
# Return whether to read the pid file only until it gives a pid
|
|
240
|
+
#
|
|
241
|
+
# True by default. Set it to false for a process whose pid changes, such
|
|
242
|
+
# as one checked for zombies.
|
|
243
|
+
#
|
|
244
|
+
# @return [Boolean]
|
|
245
|
+
#
|
|
246
|
+
# @api private
|
|
247
|
+
attr_accessor :cache_actual_pid
|
|
248
|
+
|
|
249
|
+
# Return whether bluepill also watches the process's children
|
|
250
|
+
#
|
|
251
|
+
# @return [Boolean]
|
|
252
|
+
#
|
|
253
|
+
# @api private
|
|
254
|
+
attr_accessor :monitor_children
|
|
255
|
+
|
|
256
|
+
# Return the factory that makes a Process for each child found
|
|
257
|
+
#
|
|
258
|
+
# monitor_children sets it, with the children's checks.
|
|
259
|
+
#
|
|
260
|
+
# @return [ProcessFactory, nil]
|
|
261
|
+
#
|
|
262
|
+
# @api private
|
|
263
|
+
attr_accessor :child_process_factory
|
|
264
|
+
|
|
265
|
+
# Return the command whose output is the process's pid, if any
|
|
266
|
+
#
|
|
267
|
+
# Used instead of pid_file when it's set.
|
|
268
|
+
#
|
|
269
|
+
# @return [String, nil]
|
|
270
|
+
#
|
|
271
|
+
# @api private
|
|
272
|
+
attr_accessor :pid_command
|
|
273
|
+
|
|
274
|
+
# Return whether bluepill starts the process when it loads
|
|
275
|
+
#
|
|
276
|
+
# Only false keeps it from starting: the process is then unmonitored.
|
|
277
|
+
#
|
|
278
|
+
# @return [Boolean, nil]
|
|
279
|
+
#
|
|
280
|
+
# @api private
|
|
281
|
+
attr_accessor :auto_start
|
|
282
|
+
|
|
283
|
+
# Return the extra groups the process's commands run with
|
|
284
|
+
#
|
|
285
|
+
# @return [Array<String>, nil]
|
|
286
|
+
#
|
|
287
|
+
# @api private
|
|
288
|
+
attr_accessor :supplementary_groups
|
|
289
|
+
|
|
290
|
+
# Return the signals that stop the process, with the delays between
|
|
291
|
+
#
|
|
292
|
+
# A signal, then a delay in seconds and a signal, and so on, such as
|
|
293
|
+
# [:quit, 30, :term, 5, :kill]. Each later signal is only sent if the
|
|
294
|
+
# process is still running.
|
|
295
|
+
#
|
|
296
|
+
# @return [Array<Symbol, Integer>, nil]
|
|
297
|
+
#
|
|
298
|
+
# @api private
|
|
299
|
+
attr_accessor :stop_signals
|
|
300
|
+
|
|
301
|
+
# Return the event to dispatch when the start command takes too long
|
|
302
|
+
#
|
|
303
|
+
# The default is start, which tries again.
|
|
304
|
+
#
|
|
305
|
+
# @return [String, Symbol]
|
|
306
|
+
#
|
|
307
|
+
# @api private
|
|
308
|
+
attr_accessor :on_start_timeout
|
|
309
|
+
|
|
310
|
+
# Return whether a group's start runs without waiting for this process
|
|
311
|
+
#
|
|
312
|
+
# @return [Boolean]
|
|
313
|
+
#
|
|
314
|
+
# @api private
|
|
315
|
+
attr_accessor :group_start_noblock
|
|
316
|
+
|
|
317
|
+
# Return whether a group's restart runs without waiting for this process
|
|
318
|
+
#
|
|
319
|
+
# @return [Boolean]
|
|
320
|
+
#
|
|
321
|
+
# @api private
|
|
322
|
+
attr_accessor :group_restart_noblock
|
|
323
|
+
|
|
324
|
+
# Return whether a group's stop runs without waiting for this process
|
|
325
|
+
#
|
|
326
|
+
# @return [Boolean]
|
|
327
|
+
#
|
|
328
|
+
# @api private
|
|
329
|
+
attr_accessor :group_stop_noblock
|
|
330
|
+
|
|
331
|
+
# Return whether a group's unmonitor runs without waiting for it
|
|
332
|
+
#
|
|
333
|
+
# @return [Boolean]
|
|
334
|
+
#
|
|
335
|
+
# @api private
|
|
336
|
+
attr_accessor :group_unmonitor_noblock
|
|
337
|
+
|
|
338
|
+
# Return the processes made for the process's children
|
|
339
|
+
#
|
|
340
|
+
# Only filled when monitor_children is set.
|
|
341
|
+
#
|
|
342
|
+
# @return [Array<Process>]
|
|
343
|
+
#
|
|
344
|
+
# @api private
|
|
345
|
+
attr_reader :children
|
|
346
|
+
|
|
347
|
+
# Return the process's recent events
|
|
348
|
+
#
|
|
349
|
+
# @return [ProcessStatistics]
|
|
350
|
+
#
|
|
351
|
+
# @api private
|
|
352
|
+
attr_reader :statistics
|
|
353
|
+
|
|
354
|
+
# Return the logger, prefixed with the process's name
|
|
355
|
+
#
|
|
356
|
+
# @return [Logger]
|
|
357
|
+
#
|
|
358
|
+
# @api private
|
|
359
|
+
attr_reader :logger
|
|
57
360
|
|
|
58
361
|
state_machine initial: :unmonitored do
|
|
59
362
|
# These are the idle states, i.e. only an event (either external or internal) will trigger a transition.
|
|
@@ -109,6 +412,21 @@ module Bluepill
|
|
|
109
412
|
after_transition any => any, do: :record_transition
|
|
110
413
|
end
|
|
111
414
|
|
|
415
|
+
# Create a process from its checks and options
|
|
416
|
+
#
|
|
417
|
+
# Each check is a watch, or a trigger if one has its name. Options that
|
|
418
|
+
# aren't given keep their defaults: 3 second grace times, a cached pid,
|
|
419
|
+
# and children left alone.
|
|
420
|
+
#
|
|
421
|
+
# @param [String] process_name the process's name
|
|
422
|
+
# @param [Hash{Symbol => Hash{Symbol => Object}}] checks options by
|
|
423
|
+
# check name
|
|
424
|
+
# @param [Hash{Symbol => Object}] options the process's options, such as
|
|
425
|
+
# CONFIGURABLE_ATTRIBUTES, :logger and :actual_pid
|
|
426
|
+
#
|
|
427
|
+
# @return [void]
|
|
428
|
+
#
|
|
429
|
+
# @api private
|
|
112
430
|
def initialize(process_name, checks, options = {})
|
|
113
431
|
@name = process_name
|
|
114
432
|
@event_mutex = Monitor.new
|
|
@@ -137,15 +455,24 @@ module Bluepill
|
|
|
137
455
|
@group_start_noblock = @group_stop_noblock = @group_restart_noblock = @group_unmonitor_noblock = true
|
|
138
456
|
|
|
139
457
|
CONFIGURABLE_ATTRIBUTES.each do |attribute_name|
|
|
140
|
-
|
|
458
|
+
public_send("#{attribute_name}=", options.fetch(attribute_name)) if options.key?(attribute_name)
|
|
141
459
|
end
|
|
142
460
|
|
|
143
461
|
# Let state_machine do its initialization stuff
|
|
144
462
|
super() # no arguments intentional
|
|
145
463
|
end
|
|
146
464
|
|
|
465
|
+
# Check on the process once a tick
|
|
466
|
+
#
|
|
467
|
+
# Unless ticks are being skipped, fire the state machine's tick event, and
|
|
468
|
+
# then, for a process that's up, run its watches and tick its children.
|
|
469
|
+
#
|
|
470
|
+
# @return [void]
|
|
471
|
+
#
|
|
472
|
+
# @api private
|
|
147
473
|
def tick
|
|
148
474
|
return if skipping_ticks?
|
|
475
|
+
|
|
149
476
|
self.skip_ticks_until = nil
|
|
150
477
|
|
|
151
478
|
# clear the memoization per tick
|
|
@@ -154,33 +481,61 @@ module Bluepill
|
|
|
154
481
|
# Deal with thread cleanup here since the stopping state isn't used
|
|
155
482
|
clean_threads if unmonitored?
|
|
156
483
|
|
|
157
|
-
# run state machine transitions
|
|
158
|
-
|
|
484
|
+
# run state machine transitions. state_machine defines tick in a
|
|
485
|
+
# module it creates, which the signatures can't name.
|
|
486
|
+
super # steep:ignore UnexpectedSuper
|
|
159
487
|
|
|
160
488
|
return unless up?
|
|
489
|
+
|
|
161
490
|
run_watches
|
|
162
491
|
|
|
163
492
|
return unless monitor_children?
|
|
493
|
+
|
|
164
494
|
refresh_children!
|
|
165
495
|
children.each(&:tick)
|
|
166
496
|
end
|
|
167
497
|
|
|
498
|
+
# Set the logger, for the process and for its watches and triggers
|
|
499
|
+
#
|
|
500
|
+
# @param [Logger] logger the logger to use
|
|
501
|
+
#
|
|
502
|
+
# @return [void]
|
|
503
|
+
#
|
|
504
|
+
# @api private
|
|
168
505
|
def logger=(logger)
|
|
169
506
|
@logger = logger
|
|
170
507
|
watches.each { |w| w.logger = logger }
|
|
171
508
|
triggers.each { |t| t.logger = logger }
|
|
172
509
|
end
|
|
173
510
|
|
|
174
|
-
#
|
|
511
|
+
# Record an event and fire it on the state machine
|
|
512
|
+
#
|
|
513
|
+
# @param [Symbol, String] event the event, such as :start or :restart
|
|
514
|
+
# @param [String, nil] reason why, for the process's event history
|
|
515
|
+
#
|
|
516
|
+
# @return [Boolean] whether the state machine made the transition
|
|
517
|
+
#
|
|
518
|
+
# @api private
|
|
175
519
|
def dispatch!(event, reason = nil)
|
|
176
520
|
@event_mutex.synchronize do
|
|
177
521
|
@statistics.record_event(event, reason)
|
|
178
|
-
|
|
522
|
+
public_send(event)
|
|
179
523
|
end
|
|
180
524
|
end
|
|
181
525
|
|
|
526
|
+
# Clear the watches' history after the process changes state
|
|
527
|
+
#
|
|
528
|
+
# A process that's monitoring its children also clears their list, so it
|
|
529
|
+
# finds them again. A loopback, such as up to up, changes nothing.
|
|
530
|
+
#
|
|
531
|
+
# @param [#from_name, #to_name, #loopback?] transition the transition
|
|
532
|
+
#
|
|
533
|
+
# @return [void]
|
|
534
|
+
#
|
|
535
|
+
# @api private
|
|
182
536
|
def record_transition(transition)
|
|
183
537
|
return if transition.loopback?
|
|
538
|
+
|
|
184
539
|
@transitioned = true
|
|
185
540
|
|
|
186
541
|
# When a process changes state, we should clear the memory of all the watches
|
|
@@ -194,56 +549,105 @@ module Bluepill
|
|
|
194
549
|
logger.info "Going from #{transition.from_name} => #{transition.to_name}"
|
|
195
550
|
end
|
|
196
551
|
|
|
552
|
+
# Tell every trigger about a transition
|
|
553
|
+
#
|
|
554
|
+
# @param [#from_name, #to_name, #loopback?] transition the transition
|
|
555
|
+
#
|
|
556
|
+
# @return [void]
|
|
557
|
+
#
|
|
558
|
+
# @raise [StandardError] what a trigger raised, after logging it
|
|
559
|
+
#
|
|
560
|
+
# @api private
|
|
197
561
|
def notify_triggers(transition)
|
|
198
562
|
triggers.each do |trigger|
|
|
199
563
|
begin
|
|
200
564
|
trigger.notify(transition)
|
|
201
|
-
rescue => e
|
|
565
|
+
rescue StandardError => e
|
|
202
566
|
logger.err e.backtrace
|
|
203
|
-
raise
|
|
567
|
+
raise
|
|
204
568
|
end
|
|
205
569
|
end
|
|
206
570
|
end
|
|
207
571
|
|
|
208
|
-
#
|
|
209
|
-
|
|
572
|
+
# Add a check to run on the process every interval
|
|
573
|
+
#
|
|
574
|
+
# @param [Symbol] name the check's name, such as :mem_usage
|
|
575
|
+
# @param [Hash{Symbol => Object}] options the check's options
|
|
576
|
+
#
|
|
577
|
+
# @return [Array<ConditionWatch>] the process's watches
|
|
578
|
+
#
|
|
579
|
+
# @api private
|
|
580
|
+
def add_watch(name, options)
|
|
210
581
|
watches << ConditionWatch.new(name, options.merge(logger: logger))
|
|
211
582
|
end
|
|
212
583
|
|
|
213
|
-
|
|
584
|
+
# Add a trigger to notify of the process's transitions
|
|
585
|
+
#
|
|
586
|
+
# @param [Symbol] name the trigger's name, such as :flapping
|
|
587
|
+
# @param [Hash{Symbol => Object}] options the trigger's options
|
|
588
|
+
#
|
|
589
|
+
# @return [Array<Trigger>] the process's triggers
|
|
590
|
+
#
|
|
591
|
+
# @api private
|
|
592
|
+
def add_trigger(name, options)
|
|
214
593
|
triggers << Trigger[name].new(self, options.merge(logger: logger))
|
|
215
594
|
end
|
|
216
595
|
|
|
596
|
+
# Run every watch, and dispatch the events they fire
|
|
597
|
+
#
|
|
598
|
+
# Each watch runs in a thread of its own. Once an event changes the
|
|
599
|
+
# process's state, the events after it are dropped.
|
|
600
|
+
#
|
|
601
|
+
# @return [void]
|
|
602
|
+
#
|
|
603
|
+
# @api private
|
|
217
604
|
def run_watches
|
|
218
|
-
now = Time.now
|
|
605
|
+
now = Integer(Time.now)
|
|
219
606
|
|
|
220
607
|
threads = watches.collect do |watch|
|
|
221
|
-
[watch, Thread.new {
|
|
608
|
+
[watch, Thread.new { watch.run(actual_pid, now) }] #: [ConditionWatch, Thread]
|
|
222
609
|
end
|
|
223
610
|
|
|
224
611
|
@transitioned = false
|
|
225
612
|
|
|
226
613
|
threads.each_with_object([]) do |(watch, thread), events|
|
|
227
|
-
thread.
|
|
228
|
-
next if
|
|
229
|
-
|
|
230
|
-
|
|
614
|
+
watch_events = thread.value #: Array[Symbol]
|
|
615
|
+
next if watch_events.empty?
|
|
616
|
+
|
|
617
|
+
logger.info "#{watch.name} dispatched: #{watch_events.join(',')}"
|
|
618
|
+
watch_events.each do |event|
|
|
231
619
|
events << [event, watch.to_s]
|
|
232
620
|
end
|
|
233
621
|
end.each do |event, reason| # rubocop:disable Style/MultilineBlockChain
|
|
234
622
|
break if @transitioned
|
|
623
|
+
|
|
235
624
|
dispatch!(event, reason)
|
|
236
625
|
end
|
|
237
626
|
end
|
|
238
627
|
|
|
628
|
+
# Set the state to up, down or unmonitored when bluepill loads
|
|
629
|
+
#
|
|
630
|
+
# A running process is up. Otherwise it's down, so that it starts, unless
|
|
631
|
+
# auto_start is false.
|
|
632
|
+
#
|
|
633
|
+
# @return [void]
|
|
634
|
+
#
|
|
635
|
+
# @api private
|
|
239
636
|
def determine_initial_state
|
|
240
637
|
if process_running?(true)
|
|
241
638
|
self.state = 'up'
|
|
242
639
|
return
|
|
243
640
|
end
|
|
244
|
-
self.state = (
|
|
641
|
+
self.state = auto_start.equal?(false) ? 'unmonitored' : 'down' # we need to check for false value
|
|
245
642
|
end
|
|
246
643
|
|
|
644
|
+
# Carry out a command from the command line
|
|
645
|
+
#
|
|
646
|
+
# @param [String] cmd start, stop, restart or unmonitor
|
|
647
|
+
#
|
|
648
|
+
# @return [void]
|
|
649
|
+
#
|
|
650
|
+
# @api private
|
|
247
651
|
def handle_user_command(cmd)
|
|
248
652
|
case cmd
|
|
249
653
|
when 'start'
|
|
@@ -265,209 +669,205 @@ module Bluepill
|
|
|
265
669
|
end
|
|
266
670
|
end
|
|
267
671
|
|
|
268
|
-
#
|
|
672
|
+
# Return whether the process is running, and forget its pid if not
|
|
673
|
+
#
|
|
674
|
+
# The answer is kept for the rest of the tick.
|
|
675
|
+
#
|
|
676
|
+
# @param [Boolean] force whether to check again this tick
|
|
677
|
+
#
|
|
678
|
+
# @return [Boolean]
|
|
679
|
+
#
|
|
680
|
+
# @api private
|
|
269
681
|
def process_running?(force = false)
|
|
270
682
|
@process_running = nil if force # clear existing state if forced
|
|
271
683
|
|
|
272
684
|
@process_running ||= signal_process(0)
|
|
273
685
|
# the process isn't running, so we should clear the PID
|
|
274
686
|
clear_pid unless @process_running
|
|
275
|
-
@process_running
|
|
276
|
-
end
|
|
277
|
-
|
|
278
|
-
def start_process
|
|
279
|
-
ProcessJournal.kill_all_from_journal(name) # be sure nothing else is running from previous runs
|
|
280
|
-
pre_start_process
|
|
281
|
-
logger.warning "Executing start command: #{start_command}"
|
|
282
|
-
if daemonize?
|
|
283
|
-
daemon_id = System.daemonize(start_command, system_command_options)
|
|
284
|
-
if daemon_id
|
|
285
|
-
ProcessJournal.append_pid_to_journal(name, daemon_id)
|
|
286
|
-
children.each do |child|
|
|
287
|
-
ProcessJournal.append_pid_to_journal(name, child.actual_pid)
|
|
288
|
-
end if monitor_children?
|
|
289
|
-
end
|
|
290
|
-
daemon_id
|
|
291
|
-
else
|
|
292
|
-
# This is a self-daemonizing process
|
|
293
|
-
with_timeout(start_grace_time, on_start_timeout) do
|
|
294
|
-
result = System.execute_blocking(start_command, system_command_options)
|
|
295
|
-
|
|
296
|
-
unless result[:exit_code].zero?
|
|
297
|
-
logger.warning 'Start command execution returned non-zero exit code:'
|
|
298
|
-
logger.warning result.inspect
|
|
299
|
-
end
|
|
300
|
-
end
|
|
301
|
-
end
|
|
302
|
-
|
|
303
|
-
skip_ticks_for(start_grace_time)
|
|
304
|
-
end
|
|
305
|
-
|
|
306
|
-
def pre_start_process
|
|
307
|
-
return unless pre_start_command
|
|
308
|
-
logger.warning "Executing pre start command: #{pre_start_command}"
|
|
309
|
-
result = System.execute_blocking(pre_start_command, system_command_options)
|
|
310
|
-
return if result[:exit_code].zero?
|
|
311
|
-
logger.warning 'Pre start command execution returned non-zero exit code:'
|
|
312
|
-
logger.warning result.inspect
|
|
313
|
-
end
|
|
314
|
-
|
|
315
|
-
def stop_process
|
|
316
|
-
if monitor_children
|
|
317
|
-
System.get_children(actual_pid).each do |child_pid|
|
|
318
|
-
ProcessJournal.append_pid_to_journal(name, child_pid)
|
|
319
|
-
end
|
|
320
|
-
end
|
|
321
|
-
|
|
322
|
-
if stop_command
|
|
323
|
-
cmd = prepare_command(stop_command)
|
|
324
|
-
logger.warning "Executing stop command: #{cmd}"
|
|
325
|
-
|
|
326
|
-
with_timeout(stop_grace_time, 'stop') do
|
|
327
|
-
result = System.execute_blocking(cmd, system_command_options)
|
|
328
|
-
|
|
329
|
-
unless result[:exit_code].zero?
|
|
330
|
-
logger.warning 'Stop command execution returned non-zero exit code:'
|
|
331
|
-
logger.warning result.inspect
|
|
332
|
-
end
|
|
333
|
-
end
|
|
334
|
-
cleanup_process
|
|
335
|
-
elsif stop_signals
|
|
336
|
-
# issue stop signals with configurable delay between each
|
|
337
|
-
logger.warning "Sending stop signals to #{actual_pid}"
|
|
338
|
-
@threads << Thread.new(self, stop_signals.clone) do |process, stop_signals|
|
|
339
|
-
signal = stop_signals.shift
|
|
340
|
-
logger.info "Sending signal #{signal} to #{process.actual_pid}"
|
|
341
|
-
process.signal_process(signal) # send first signal
|
|
342
|
-
|
|
343
|
-
until stop_signals.empty?
|
|
344
|
-
# we already checked to make sure stop_signals had an odd number of items
|
|
345
|
-
delay = stop_signals.shift
|
|
346
|
-
signal = stop_signals.shift
|
|
347
|
-
|
|
348
|
-
logger.debug "Sleeping for #{delay} seconds"
|
|
349
|
-
sleep delay
|
|
350
|
-
# break unless signal_process(0) #break unless the process can be reached
|
|
351
|
-
unless process.signal_process(0)
|
|
352
|
-
logger.debug 'Process has terminated.'
|
|
353
|
-
break
|
|
354
|
-
end
|
|
355
|
-
logger.info "Sending signal #{signal} to #{process.actual_pid}"
|
|
356
|
-
process.signal_process(signal)
|
|
357
|
-
end
|
|
358
|
-
cleanup_process
|
|
359
|
-
end
|
|
360
|
-
else
|
|
361
|
-
logger.warning "Executing default stop command. Sending TERM signal to #{actual_pid}"
|
|
362
|
-
signal_process('TERM')
|
|
363
|
-
cleanup_process
|
|
364
|
-
end
|
|
365
|
-
|
|
366
|
-
skip_ticks_for(stop_grace_time)
|
|
367
|
-
end
|
|
368
|
-
|
|
369
|
-
def restart_process
|
|
370
|
-
if restart_command
|
|
371
|
-
cmd = prepare_command(restart_command)
|
|
372
|
-
|
|
373
|
-
logger.warning "Executing restart command: #{cmd}"
|
|
374
|
-
|
|
375
|
-
with_timeout(restart_grace_time, 'restart') do
|
|
376
|
-
result = System.execute_blocking(cmd, system_command_options)
|
|
377
|
-
|
|
378
|
-
unless result[:exit_code].zero?
|
|
379
|
-
logger.warning 'Restart command execution returned non-zero exit code:'
|
|
380
|
-
logger.warning result.inspect
|
|
381
|
-
end
|
|
382
|
-
end
|
|
383
|
-
|
|
384
|
-
skip_ticks_for(restart_grace_time)
|
|
385
|
-
else
|
|
386
|
-
logger.warning 'No restart_command specified. Must stop and start to restart'
|
|
387
|
-
stop_process
|
|
388
|
-
start_process
|
|
389
|
-
end
|
|
390
|
-
end
|
|
391
|
-
|
|
392
|
-
def cleanup_process
|
|
393
|
-
ProcessJournal.kill_all_from_journal(name) # finish cleanup
|
|
394
|
-
unlink_pid # TODO: we only write the pid file if we daemonize, should we only unlink it if we daemonize?
|
|
687
|
+
@process_running #: bool
|
|
395
688
|
end
|
|
396
689
|
|
|
690
|
+
# Kill the threads the process's commands left running
|
|
691
|
+
#
|
|
692
|
+
# @return [void]
|
|
693
|
+
#
|
|
694
|
+
# @api private
|
|
397
695
|
def clean_threads
|
|
398
696
|
@threads.each(&:kill)
|
|
399
697
|
@threads.clear
|
|
400
698
|
end
|
|
401
699
|
|
|
700
|
+
# Return whether bluepill daemonizes the process
|
|
701
|
+
#
|
|
702
|
+
# @return [Boolean]
|
|
703
|
+
#
|
|
704
|
+
# @api private
|
|
402
705
|
def daemonize?
|
|
403
706
|
!!daemonize
|
|
404
707
|
end
|
|
405
708
|
|
|
709
|
+
# Return whether bluepill watches the process's children
|
|
710
|
+
#
|
|
711
|
+
# @return [Boolean]
|
|
712
|
+
#
|
|
713
|
+
# @api private
|
|
406
714
|
def monitor_children?
|
|
407
715
|
!!monitor_children
|
|
408
716
|
end
|
|
409
717
|
|
|
718
|
+
# Send a signal to the process
|
|
719
|
+
#
|
|
720
|
+
# A process without a pid isn't running, so it gets no signal and nothing
|
|
721
|
+
# is logged. A failure is logged.
|
|
722
|
+
#
|
|
723
|
+
# @param [Integer, String, Symbol] code the signal, such as 0 or :term
|
|
724
|
+
#
|
|
725
|
+
# @return [Boolean] whether the signal was sent
|
|
726
|
+
#
|
|
727
|
+
# @api private
|
|
410
728
|
def signal_process(code)
|
|
411
|
-
code = code.to_s.upcase if code.is_a?(String) || code.
|
|
412
|
-
|
|
729
|
+
code = code.to_s.upcase if code.is_a?(String) || code.instance_of?(Symbol)
|
|
730
|
+
pid = actual_pid
|
|
731
|
+
# Without a pid the process isn't running. That's expected before the
|
|
732
|
+
# first start, and actual_pid already warns about a missing pid file.
|
|
733
|
+
return false unless pid
|
|
734
|
+
|
|
735
|
+
# kill with a pid of 0 or less signals a whole process group, which
|
|
736
|
+
# could include bluepill itself, so only signal a real pid
|
|
737
|
+
raise "invalid pid #{pid}" unless pid > 0
|
|
738
|
+
|
|
739
|
+
::Process.kill(code, pid)
|
|
413
740
|
true
|
|
414
|
-
rescue => e
|
|
415
|
-
logger.err "Failed to signal process #{
|
|
741
|
+
rescue StandardError => e
|
|
742
|
+
logger.err "Failed to signal process #{pid} with code #{code}: #{e}"
|
|
416
743
|
false
|
|
417
744
|
end
|
|
418
745
|
|
|
746
|
+
# Return whether the pid is kept once it's read
|
|
747
|
+
#
|
|
748
|
+
# @return [Boolean]
|
|
749
|
+
#
|
|
750
|
+
# @api private
|
|
419
751
|
def cache_actual_pid?
|
|
420
752
|
!!@cache_actual_pid
|
|
421
753
|
end
|
|
422
754
|
|
|
755
|
+
# Return the process's pid, from pid_command or the pid file
|
|
756
|
+
#
|
|
757
|
+
# @return [Integer, nil] the pid, or nil if there's none yet
|
|
758
|
+
#
|
|
759
|
+
# @api private
|
|
423
760
|
def actual_pid
|
|
424
761
|
pid_command ? pid_from_command : pid_from_file
|
|
425
762
|
end
|
|
426
763
|
|
|
764
|
+
# Return the pid in the pid file, cached unless cache_actual_pid is false
|
|
765
|
+
#
|
|
766
|
+
# A missing pid file is logged as a warning.
|
|
767
|
+
#
|
|
768
|
+
# @return [Integer, nil] the pid, or nil without a pid file or a pid
|
|
769
|
+
#
|
|
770
|
+
# @api private
|
|
427
771
|
def pid_from_file
|
|
428
772
|
return @actual_pid if cache_actual_pid? && @actual_pid
|
|
429
|
-
@actual_pid = begin
|
|
430
|
-
if pid_file
|
|
431
|
-
if File.exist?(pid_file)
|
|
432
|
-
str = File.read(pid_file)
|
|
433
|
-
str.to_i unless str.empty?
|
|
434
|
-
else
|
|
435
|
-
logger.warning("pid_file #{pid_file} does not exist or cannot be read")
|
|
436
|
-
nil
|
|
437
|
-
end
|
|
438
|
-
end
|
|
439
|
-
end
|
|
440
|
-
end
|
|
441
773
|
|
|
774
|
+
@actual_pid = if pid_file
|
|
775
|
+
if File.exist?(pid_file)
|
|
776
|
+
positive_pid(File.read(pid_file).to_i)
|
|
777
|
+
else
|
|
778
|
+
logger.warning("pid_file #{pid_file} does not exist or cannot be read")
|
|
779
|
+
nil
|
|
780
|
+
end
|
|
781
|
+
end
|
|
782
|
+
end
|
|
783
|
+
|
|
784
|
+
# Return the pid that pid_command prints
|
|
785
|
+
#
|
|
786
|
+
# Output that isn't only digits gives no pid.
|
|
787
|
+
#
|
|
788
|
+
# @return [Integer, nil] the pid, or nil if the output isn't one
|
|
789
|
+
#
|
|
790
|
+
# @api private
|
|
442
791
|
def pid_from_command
|
|
443
|
-
|
|
444
|
-
(
|
|
445
|
-
end
|
|
446
|
-
|
|
792
|
+
# Output that is not just digits reads as nil, so as 0
|
|
793
|
+
positive_pid(`#{pid_command}`.strip[/\A\d+\z/].to_i)
|
|
794
|
+
end
|
|
795
|
+
|
|
796
|
+
# Return the pid, unless it's 0 or less
|
|
797
|
+
#
|
|
798
|
+
# An empty or garbled pid file reads as 0, which is never a real process.
|
|
799
|
+
#
|
|
800
|
+
# @param [Integer] pid the pid that was read
|
|
801
|
+
#
|
|
802
|
+
# @return [Integer, nil] the pid, or nil if it's not a real one
|
|
803
|
+
#
|
|
804
|
+
# @api private
|
|
805
|
+
def positive_pid(pid)
|
|
806
|
+
pid if pid > 0
|
|
807
|
+
end
|
|
808
|
+
|
|
809
|
+
# Set the pid, and add it to the process's journal
|
|
810
|
+
#
|
|
811
|
+
# @param [Integer] pid the process's pid
|
|
812
|
+
#
|
|
813
|
+
# @return [void]
|
|
814
|
+
#
|
|
815
|
+
# @api private
|
|
447
816
|
def actual_pid=(pid)
|
|
448
817
|
ProcessJournal.append_pid_to_journal(name, pid) # be sure to always log the pid
|
|
449
818
|
@actual_pid = pid
|
|
450
819
|
end
|
|
451
820
|
|
|
821
|
+
# Forget the process's pid
|
|
822
|
+
#
|
|
823
|
+
# @return [nil]
|
|
824
|
+
#
|
|
825
|
+
# @api private
|
|
452
826
|
def clear_pid
|
|
453
827
|
@actual_pid = nil
|
|
454
828
|
end
|
|
455
829
|
|
|
830
|
+
# Delete the pid file, if there is one
|
|
831
|
+
#
|
|
832
|
+
# @return [void]
|
|
833
|
+
#
|
|
834
|
+
# @api private
|
|
456
835
|
def unlink_pid
|
|
457
836
|
System.delete_if_exists(pid_file)
|
|
458
837
|
end
|
|
459
838
|
|
|
460
|
-
#
|
|
839
|
+
# Skip ticks for a while, such as for a command's grace time
|
|
840
|
+
#
|
|
841
|
+
# The times add up, so a stop followed by a start, which is how
|
|
842
|
+
# restart_process works without a restart_command, waits out both grace
|
|
843
|
+
# times.
|
|
844
|
+
#
|
|
845
|
+
# @param [Integer, Float] seconds how long to skip ticks for
|
|
846
|
+
#
|
|
847
|
+
# @return [Integer] when ticks resume, in epoch seconds
|
|
848
|
+
#
|
|
849
|
+
# @api private
|
|
461
850
|
def skip_ticks_for(seconds)
|
|
462
|
-
|
|
463
|
-
# i.e. if two calls for skip_ticks_for come in for 5 and 10, should it skip for 10 or 15?
|
|
464
|
-
self.skip_ticks_until = (skip_ticks_until || Time.now.to_i) + seconds.to_i
|
|
851
|
+
self.skip_ticks_until = (skip_ticks_until || Integer(Time.now)) + Integer(seconds)
|
|
465
852
|
end
|
|
466
853
|
|
|
854
|
+
# Return whether ticks are being skipped
|
|
855
|
+
#
|
|
856
|
+
# @return [Boolean, nil]
|
|
857
|
+
#
|
|
858
|
+
# @api private
|
|
467
859
|
def skipping_ticks?
|
|
468
|
-
skip_ticks_until && skip_ticks_until > Time.now
|
|
860
|
+
skip_ticks_until && skip_ticks_until > Integer(Time.now)
|
|
469
861
|
end
|
|
470
862
|
|
|
863
|
+
# Update the list of children from the tick's ps table
|
|
864
|
+
#
|
|
865
|
+
# Children that are no longer running are dropped, and a Process is made
|
|
866
|
+
# for each new one, with the checks monitor_children gave.
|
|
867
|
+
#
|
|
868
|
+
# @return [void]
|
|
869
|
+
#
|
|
870
|
+
# @api private
|
|
471
871
|
def refresh_children!
|
|
472
872
|
# First prune the list of dead children
|
|
473
873
|
@children.delete_if { |child| !child.process_running?(true) }
|
|
@@ -475,48 +875,19 @@ module Bluepill
|
|
|
475
875
|
# Add new found children to the list
|
|
476
876
|
new_children_pids = System.get_children(actual_pid) - @children.collect(&:actual_pid)
|
|
477
877
|
|
|
478
|
-
unless new_children_pids.empty?
|
|
479
|
-
logger.info "Existing children: #{@children.collect(&:actual_pid).join(',')}. Got new children: #{new_children_pids.inspect} for #{actual_pid}"
|
|
480
|
-
end
|
|
878
|
+
logger.info "Existing children: #{@children.collect(&:actual_pid).join(',')}. Got new children: #{new_children_pids} for #{actual_pid}" unless new_children_pids.empty?
|
|
481
879
|
|
|
482
880
|
# Construct a new process wrapper for each new found children
|
|
881
|
+
# monitor_children sets the factory
|
|
882
|
+
factory = child_process_factory #: ProcessFactory
|
|
483
883
|
new_children_pids.each do |child_pid|
|
|
484
884
|
ProcessJournal.append_pid_to_journal(name, child_pid) if daemonize?
|
|
485
885
|
child_name = "<child(pid:#{child_pid})>"
|
|
486
|
-
|
|
886
|
+
child_logger = logger.prefix_with(child_name)
|
|
487
887
|
|
|
488
|
-
child =
|
|
888
|
+
child = factory.create_child_process(child_name, child_pid, child_logger)
|
|
489
889
|
@children << child
|
|
490
890
|
end
|
|
491
891
|
end
|
|
492
|
-
|
|
493
|
-
def prepare_command(command)
|
|
494
|
-
command.to_s.gsub('{{PID}}', actual_pid.to_s)
|
|
495
|
-
end
|
|
496
|
-
|
|
497
|
-
def system_command_options
|
|
498
|
-
{
|
|
499
|
-
uid: uid,
|
|
500
|
-
gid: gid,
|
|
501
|
-
working_dir: working_dir,
|
|
502
|
-
environment: environment,
|
|
503
|
-
pid_file: pid_file,
|
|
504
|
-
logger: logger,
|
|
505
|
-
stdin: stdin,
|
|
506
|
-
stdout: stdout,
|
|
507
|
-
stderr: stderr,
|
|
508
|
-
supplementary_groups: supplementary_groups,
|
|
509
|
-
}
|
|
510
|
-
end
|
|
511
|
-
|
|
512
|
-
def with_timeout(secs, next_state = nil, &blk)
|
|
513
|
-
# Attempt to execute the passed block. If the block takes
|
|
514
|
-
# too long, transition to the indicated next state.
|
|
515
|
-
Timeout.timeout(secs.to_f, &blk)
|
|
516
|
-
rescue Timeout::Error
|
|
517
|
-
logger.err 'Execution is taking longer than expected.'
|
|
518
|
-
logger.err 'Did you forget to tell bluepill to daemonize this process?'
|
|
519
|
-
dispatch!(next_state)
|
|
520
|
-
end
|
|
521
892
|
end
|
|
522
893
|
end
|