bluepill 0.1.3 → 0.1.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (43) hide show
  1. checksums.yaml +5 -5
  2. data/CHANGELOG.md +835 -0
  3. data/CONTRIBUTING.md +31 -0
  4. data/LICENSE +1 -1
  5. data/README.md +66 -29
  6. data/bin/bluepill +1 -1
  7. data/bluepill.gemspec +17 -6
  8. data/lib/bluepill/application.rb +321 -46
  9. data/lib/bluepill/condition_watch.rb +103 -8
  10. data/lib/bluepill/controller.rb +124 -13
  11. data/lib/bluepill/dsl/app_proxy.rb +102 -3
  12. data/lib/bluepill/dsl/process_factory.rb +183 -60
  13. data/lib/bluepill/dsl/process_proxy.rb +132 -14
  14. data/lib/bluepill/dsl.rb +27 -5
  15. data/lib/bluepill/group.rb +142 -9
  16. data/lib/bluepill/logger.rb +122 -5
  17. data/lib/bluepill/process.rb +575 -201
  18. data/lib/bluepill/process_commands.rb +264 -0
  19. data/lib/bluepill/process_conditions/always_true.rb +18 -4
  20. data/lib/bluepill/process_conditions/cpu_usage.rb +34 -0
  21. data/lib/bluepill/process_conditions/file_time.rb +40 -4
  22. data/lib/bluepill/process_conditions/http.rb +81 -23
  23. data/lib/bluepill/process_conditions/mem_usage.rb +61 -2
  24. data/lib/bluepill/process_conditions/process_condition.rb +43 -0
  25. data/lib/bluepill/process_conditions/running_time.rb +36 -1
  26. data/lib/bluepill/process_conditions/zombie_process.rb +26 -5
  27. data/lib/bluepill/process_conditions.rb +15 -1
  28. data/lib/bluepill/process_journal.rb +374 -114
  29. data/lib/bluepill/process_statistics.rb +33 -3
  30. data/lib/bluepill/socket.rb +63 -16
  31. data/lib/bluepill/system/commands.rb +286 -0
  32. data/lib/bluepill/system.rb +223 -228
  33. data/lib/bluepill/trigger.rb +110 -14
  34. data/lib/bluepill/triggers/flapping.rb +84 -3
  35. data/lib/bluepill/util/rotational_array.rb +24 -1
  36. data/lib/bluepill/version.rb +71 -19
  37. data/lib/bluepill.rb +25 -2
  38. data/sig/bluepill.rbs +941 -0
  39. data/sig/manifest.yaml +7 -0
  40. metadata +16 -34
  41. data/bin/sample_forking_server +0 -66
  42. data/lib/bluepill/application/client.rb +0 -6
  43. data/lib/bluepill/application/server.rb +0 -21
@@ -3,43 +3,154 @@ require 'bluepill/system'
3
3
  require 'bluepill/process_journal'
4
4
 
5
5
  module Bluepill
6
+ # The bluepill daemon for one application
7
+ #
8
+ # It holds the application's groups of processes, ticks them every
9
+ # second, and answers the commands that the CLI sends over its socket.
10
+ #
11
+ # @api private
6
12
  class Application
13
+ # The commands that act on a process or group, and that the CLI sends
7
14
  PROCESS_COMMANDS = [:start, :stop, :restart, :unmonitor, :status].freeze
8
15
 
9
- attr_accessor :name, :logger, :base_dir, :socket, :pid_file, :kill_timeout
10
- attr_accessor :groups, :work_queue
11
- attr_accessor :pids_dir, :log_file
12
-
16
+ attr_accessor :name, :logger, :base_dir, :socket, :pid_file, :kill_timeout, :groups, :work_queue, :pids_dir, :log_file
17
+
18
+ # @!attribute [rw] name
19
+ # The application's name, which names its socket and pid file
20
+ #
21
+ # @return [String]
22
+ #
23
+ # @api private
24
+
25
+ # @!attribute [rw] logger
26
+ # The application's logger, prefixed with its name
27
+ #
28
+ # @return [Logger]
29
+ #
30
+ # @api private
31
+
32
+ # @!attribute [rw] base_dir
33
+ # The directory with the daemon's sockets, pid files and journals
34
+ #
35
+ # @return [String]
36
+ #
37
+ # @api private
38
+
39
+ # @!attribute [rw] socket
40
+ # The socket the daemon listens for commands on, once it's started
41
+ #
42
+ # @return [UNIXServer, nil]
43
+ #
44
+ # @api private
45
+
46
+ # @!attribute [rw] pid_file
47
+ # The daemon's own pid file
48
+ #
49
+ # @return [String]
50
+ #
51
+ # @api private
52
+
53
+ # @!attribute [rw] kill_timeout
54
+ # How many half seconds to wait for a previous daemon to exit
55
+ #
56
+ # @return [Integer]
57
+ #
58
+ # @api private
59
+
60
+ # @!attribute [rw] groups
61
+ # The application's groups by name, with nil for processes in no group
62
+ #
63
+ # @return [Hash{String, nil => Group}]
64
+ #
65
+ # @api private
66
+
67
+ # @!attribute [rw] work_queue
68
+ # An attribute that bluepill doesn't use
69
+ #
70
+ # @return [Object]
71
+ #
72
+ # @api private
73
+
74
+ # @!attribute [rw] pids_dir
75
+ # The directory with the pid files of the application's processes
76
+ #
77
+ # @return [String]
78
+ #
79
+ # @api private
80
+
81
+ # @!attribute [rw] log_file
82
+ # The file the daemon logs to, or nil to log to syslog
83
+ #
84
+ # @return [String, nil]
85
+ #
86
+ # @api private
87
+
88
+ # Create the daemon for an application
89
+ #
90
+ # It sets up signal traps, the logger and the pids directory. It exits
91
+ # with status 5 if it can't write to its log file or base directory.
92
+ #
93
+ # @param [String] name the application's name
94
+ # @param [Hash{Symbol => Object}] options :foreground, :log_file,
95
+ # :base_dir and :kill_timeout
96
+ #
97
+ # @return [void]
98
+ #
99
+ # @api private
13
100
  def initialize(name, options = {})
14
101
  self.name = name
15
102
 
16
103
  @foreground = options[:foreground]
17
104
  self.log_file = options[:log_file]
18
- self.base_dir = ProcessJournal.base_dir = options[:base_dir] ||
19
- ENV['BLUEPILL_BASE_DIR'] ||
20
- (::Process.euid != 0 ? File.join(ENV['HOME'], '.bluepill') : '/var/run/bluepill')
21
- self.pid_file = File.join(base_dir, 'pids', self.name + '.pid')
22
- self.pids_dir = File.join(base_dir, 'pids', self.name)
105
+ # Only used for a non-root user, who has a home
106
+ home = ENV['HOME'] #: String
107
+ self.base_dir = options[:base_dir] ||
108
+ ENV['BLUEPILL_BASE_DIR'] ||
109
+ (::Process.euid.zero? ? '/var/run/bluepill' : File.join(home, '.bluepill'))
110
+ ProcessJournal.setup(base_dir, name)
111
+ self.pid_file = File.join(base_dir, 'pids', "#{name}.pid")
112
+ self.pids_dir = File.join(base_dir, 'pids', name)
23
113
  self.kill_timeout = options[:kill_timeout] || 10
24
114
 
25
115
  self.groups = {}
26
116
 
27
- self.logger = ProcessJournal.logger = Bluepill::Logger.new(log_file: log_file, stdout: foreground?).prefix_with(self.name)
28
-
29
117
  setup_signal_traps
30
- setup_pids_dir
118
+ setup_logger_and_pids_dir
31
119
 
32
120
  @mutex = Mutex.new
33
121
  end
34
122
 
123
+ # Return whether the daemon runs in the foreground
124
+ #
125
+ # @return [Boolean]
126
+ #
127
+ # @api private
35
128
  def foreground?
36
129
  !!@foreground
37
130
  end
38
131
 
39
- def mutex(&b)
40
- @mutex.synchronize(&b)
132
+ # Run a block while holding the daemon's lock
133
+ #
134
+ # Ticks and commands from the CLI hold it, so they never overlap.
135
+ #
136
+ # @yield the work to do while holding the lock
137
+ # @yieldreturn [Object] what the block returns
138
+ #
139
+ # @return [Object] what the block returns
140
+ #
141
+ # @api private
142
+ def mutex(&block)
143
+ @mutex.synchronize(&block)
41
144
  end
42
145
 
146
+ # Start the daemon and monitor the application's processes
147
+ #
148
+ # It prints what went wrong, and exits with status 5, if the daemon
149
+ # fails to start.
150
+ #
151
+ # @return [void]
152
+ #
153
+ # @api private
43
154
  def load
44
155
  start_server
45
156
  rescue StandardError => e
@@ -49,65 +160,157 @@ module Bluepill
49
160
  exit(5)
50
161
  end
51
162
 
163
+ # @!method start(group_name = nil, process_name = nil)
164
+ # Start a process or group, or every process without either
165
+ #
166
+ # @param [String, nil] group_name the group, or the process's name
167
+ # when no group has it
168
+ # @param [String, nil] process_name the process
169
+ #
170
+ # @return [Array<String>] the affected processes, as group:process
171
+ #
172
+ # @api private
173
+
174
+ # @!method stop(group_name = nil, process_name = nil)
175
+ # Stop a process or group, or every process without either
176
+ #
177
+ # @param [String, nil] group_name the group, or the process's name
178
+ # when no group has it
179
+ # @param [String, nil] process_name the process
180
+ #
181
+ # @return [Array<String>] the affected processes, as group:process
182
+ #
183
+ # @api private
184
+
185
+ # @!method restart(group_name = nil, process_name = nil)
186
+ # Restart a process or group, or every process without either
187
+ #
188
+ # @param [String, nil] group_name the group, or the process's name
189
+ # when no group has it
190
+ # @param [String, nil] process_name the process
191
+ #
192
+ # @return [Array<String>] the affected processes, as group:process
193
+ #
194
+ # @api private
195
+
196
+ # @!method unmonitor(group_name = nil, process_name = nil)
197
+ # Stop monitoring a process or group, or every process without either
198
+ #
199
+ # @param [String, nil] group_name the group, or the process's name
200
+ # when no group has it
201
+ # @param [String, nil] process_name the process
202
+ #
203
+ # @return [Array<String>] the affected processes, as group:process
204
+ #
205
+ # @api private
206
+
207
+ # @!method status(group_name = nil, process_name = nil)
208
+ # Return the status of a process or group, or of every process
209
+ #
210
+ # @param [String, nil] group_name the group, or the process's name
211
+ # when no group has it
212
+ # @param [String, nil] process_name the process
213
+ #
214
+ # @return [Array<String>] the status lines
215
+ #
216
+ # @api private
217
+
218
+ # Define a method for each of the PROCESS_COMMANDS
52
219
  PROCESS_COMMANDS.each do |command|
53
- class_eval <<-END
220
+ class_eval <<-RUBY, __FILE__, __LINE__ + 1
54
221
  def #{command}(group_name = nil, process_name = nil)
55
222
  self.send_to_process_or_group(:#{command}, group_name, process_name)
56
223
  end
57
- END
224
+ RUBY
58
225
  end
59
226
 
227
+ # Add a process to a group, which is created if it's new
228
+ #
229
+ # @param [Process] process the process
230
+ # @param [#to_s, nil] group_name the group, or nil for no group
231
+ #
232
+ # @return [Array<Process>] the group's processes
233
+ #
234
+ # @api private
60
235
  def add_process(process, group_name = nil)
61
236
  group_name = group_name.to_s if group_name
62
237
 
63
238
  groups[group_name] ||= Group.new(group_name, logger: logger.prefix_with(group_name))
64
- groups[group_name].add_process(process)
239
+ groups.fetch(group_name).add_process(process)
65
240
  end
66
241
 
242
+ # Return the daemon's version, which the CLI checks against its own
243
+ #
244
+ # @return [Module] the Version module
245
+ #
246
+ # @api private
67
247
  def version
68
- Bluepill::Version
248
+ Version
69
249
  end
70
250
 
71
251
  protected
72
252
 
253
+ # Send a command to a process, a group, or every process
254
+ #
255
+ # @param [Symbol] method the command
256
+ # @param [String, nil] group_name the group, or the start of a process
257
+ # name that contains a colon
258
+ # @param [String, nil] process_name the process
259
+ #
260
+ # @return [Array<String>] what the groups returned, flattened
261
+ #
262
+ # @api private
73
263
  def send_to_process_or_group(method, group_name, process_name)
74
264
  if group_name.nil? && process_name.nil?
75
265
  groups.values.collect do |group|
76
- group.send(method)
266
+ group.public_send(method)
77
267
  end.flatten
78
268
  elsif groups.key?(group_name)
79
- groups[group_name].send(method, process_name)
80
- elsif process_name.nil?
81
- # they must be targeting just by process name
82
- process_name = group_name
269
+ groups.fetch(group_name).public_send(method, process_name)
270
+ else
271
+ # they must be targeting just by process name, which may contain ':'
272
+ process_name = [group_name, process_name].compact.join(':')
83
273
  groups.values.collect do |group|
84
- group.send(method, process_name)
274
+ group.public_send(method, process_name)
85
275
  end.flatten
86
- else
87
- []
88
276
  end
89
277
  end
90
278
 
279
+ # Start the thread that answers commands on the socket
280
+ #
281
+ # It kills the thread that a previous call started. Each command is a
282
+ # line, such as start:group:process, and the answer is the command's
283
+ # result, or the error it raised, marshaled.
284
+ #
285
+ # @return [Thread] the listener thread
286
+ #
287
+ # @api private
91
288
  def start_listener
92
- @listener_thread.kill if @listener_thread
289
+ previous_thread = @listener_thread
290
+ previous_thread.kill if previous_thread
93
291
  @listener_thread = Thread.new do
94
292
  loop do
293
+ client = nil #: UNIXSocket?
95
294
  begin
96
- client = socket.accept
97
- client.close_on_exec = true if client.respond_to?(:close_on_exec=)
98
- command, *args = client.readline.strip.split(':')
295
+ server = socket #: UNIXServer
296
+ client = server.accept
297
+ client.close_on_exec = true if client.respond_to?(:close_on_exec=)
298
+ # The target is group:process and the process name may contain ':'
299
+ command, *args = client.readline.strip.split(':', 3).reject(&:empty?)
99
300
  response = begin
100
- mutex { send(command, *args) }
101
- rescue => e
301
+ # An empty line leaves command nil, so public_send raises an error for the client
302
+ mutex { public_send(command, *args) } # steep:ignore ArgumentTypeMismatch
303
+ rescue StandardError => e
102
304
  e
103
305
  end
104
306
  client.write(Marshal.dump(response))
105
307
  rescue StandardError => e
106
308
  logger.err(format('Got exception in cmd listener: %s `%s`', e.class.name, e.message))
107
- e.backtrace.each { |l| logger.err(l) }
309
+ Array(e.backtrace).each { |l| logger.err(l) }
108
310
  ensure
109
311
  begin
110
- client.close
312
+ # Ruby before 3.0 never counts branches in ensure as covered
313
+ client && client.close
111
314
  rescue IOError
112
315
  # closed stream
113
316
  end
@@ -116,9 +319,19 @@ module Bluepill
116
319
  end
117
320
  end
118
321
 
322
+ # Start the daemon
323
+ #
324
+ # It stops a previous daemon for the application, clears journal locks
325
+ # it left behind, daemonizes unless it runs in the foreground, opens the
326
+ # socket, and runs until a signal stops it. Processes that are already
327
+ # running, from their pid files, are monitored again, not restarted, so
328
+ # bluepill can be upgraded without restarting them.
329
+ #
330
+ # @return [void]
331
+ #
332
+ # @api private
119
333
  def start_server
120
334
  kill_previous_bluepill
121
- ProcessJournal.kill_all_from_all_journals
122
335
  ProcessJournal.clear_all_atomic_fs_locks
123
336
 
124
337
  begin
@@ -126,34 +339,49 @@ module Bluepill
126
339
  rescue Errno::EPERM
127
340
  end
128
341
 
129
- Daemonize.daemonize unless foreground?
342
+ unless foreground?
343
+ Daemonize.daemonize
344
+ # Daemonizing ignores HUP, so trap it again to reopen the log on HUP
345
+ setup_signal_traps
346
+ end
130
347
 
131
348
  logger.reopen
132
349
 
133
350
  $0 = "bluepilld: #{name}"
134
351
 
135
- groups.each { |_, group| group.determine_initial_state }
352
+ groups.each_value(&:determine_initial_state)
136
353
 
137
354
  write_pid_file
138
- self.socket = Bluepill::Socket.server(base_dir, name)
355
+ self.socket = Socket.server(base_dir, name)
139
356
  start_listener
140
357
 
141
358
  run
142
359
  end
143
360
 
361
+ # Tick every group once a second, until a signal stops the daemon
362
+ #
363
+ # @return [void]
364
+ #
365
+ # @api private
144
366
  def run
145
367
  @running = true # set to false by signal trap
146
368
  while @running
147
369
  mutex do
148
370
  System.reset_data
149
- groups.each { |_, group| group.tick }
371
+ groups.each_value(&:tick)
150
372
  end
151
373
  sleep 1
152
374
  end
153
375
  end
154
376
 
377
+ # Clear the app's journal locks, and remove the socket and pid file
378
+ #
379
+ # The processes keep running, so the next daemon monitors them again.
380
+ #
381
+ # @return [void]
382
+ #
383
+ # @api private
155
384
  def cleanup
156
- ProcessJournal.kill_all_from_all_journals
157
385
  ProcessJournal.clear_all_atomic_fs_locks
158
386
  begin
159
387
  System.delete_if_exists(socket.path) if socket
@@ -162,12 +390,17 @@ module Bluepill
162
390
  System.delete_if_exists(pid_file)
163
391
  end
164
392
 
393
+ # Trap TERM and INT to clean up and stop, and HUP to reopen the log
394
+ #
395
+ # @return [void]
396
+ #
397
+ # @api private
165
398
  def setup_signal_traps
166
399
  terminator = proc do
167
400
  puts 'Terminating...'
168
401
  cleanup
169
402
  @running = false
170
- end
403
+ end #: ^(Integer signal) -> void
171
404
 
172
405
  Signal.trap('TERM', &terminator)
173
406
  Signal.trap('INT', &terminator)
@@ -177,24 +410,61 @@ module Bluepill
177
410
  end
178
411
  end
179
412
 
413
+ # Create the logger and the pids directory
414
+ #
415
+ # Bluepill opens its log file and creates its pids dir as the user that
416
+ # runs it. The uid and gid in the config only apply to its processes.
417
+ # It exits with status 5 if it can't.
418
+ #
419
+ # @return [void]
420
+ #
421
+ # @api private
422
+ def setup_logger_and_pids_dir
423
+ self.logger = ProcessJournal.logger = Logger.new(log_file: log_file, stdout: foreground?).prefix_with(name)
424
+ setup_pids_dir
425
+ rescue SystemCallError => e
426
+ $stderr.puts("Failed to start bluepill: #{e}")
427
+ message = 'The user running bluepill must be able to write to its log_file and base_dir. ' \
428
+ 'The uid and gid in the config only apply to the processes it monitors.'
429
+ $stderr.puts(message)
430
+ exit(5)
431
+ end
432
+
433
+ # Create the pids directory, and let every user write to it
434
+ #
435
+ # @return [void]
436
+ #
437
+ # @api private
180
438
  def setup_pids_dir
181
439
  FileUtils.mkdir_p(pids_dir) unless File.exist?(pids_dir)
182
440
  # we need everybody to be able to write to the pids_dir as processes managed by
183
441
  # bluepill will be writing to this dir after they've dropped privileges
184
- FileUtils.chmod(0777, pids_dir)
442
+ FileUtils.chmod(0o777, pids_dir)
185
443
  end
186
444
 
445
+ # Interrupt a daemon for this application that's still running
446
+ #
447
+ # It waits up to kill_timeout half seconds for the daemon to exit, and
448
+ # exits with status 4 if it doesn't, or if signaling it fails.
449
+ #
450
+ # @return [void]
451
+ #
452
+ # @api private
187
453
  def kill_previous_bluepill
188
454
  return unless File.exist?(pid_file)
455
+
456
+ # An empty or garbled pid file reads as 0, and signaling pid 0 would
457
+ # interrupt bluepill's own process group
189
458
  previous_pid = File.read(pid_file).to_i
190
- return unless System.pid_alive?(previous_pid)
459
+ return unless previous_pid > 0 && System.pid_alive?(previous_pid)
460
+
191
461
  ::Process.kill(0, previous_pid)
192
462
  puts "Killing previous bluepilld[#{previous_pid}]"
193
463
  ::Process.kill(2, previous_pid)
194
- rescue => e
464
+ rescue StandardError => e
195
465
  $stderr.puts 'Encountered error trying to kill previous bluepill:'
196
- $stderr.puts "#{e.class}: #{e.message}"
197
- exit(4) unless e.is_a?(Errno::ESRCH)
466
+ $stderr.puts "#{e.class}: #{e}"
467
+ exit(4) unless e.instance_of?(Errno::ESRCH)
198
468
  else
199
469
  kill_timeout.times do |_i|
200
470
  sleep 0.5
@@ -207,6 +477,11 @@ module Bluepill
207
477
  end
208
478
  end
209
479
 
480
+ # Write the daemon's pid to its pid file
481
+ #
482
+ # @return [void]
483
+ #
484
+ # @api private
210
485
  def write_pid_file
211
486
  File.open(pid_file, 'w') { |x| x.write(::Process.pid) }
212
487
  end
@@ -1,18 +1,76 @@
1
1
  module Bluepill
2
+ # One entry of a watch's history: the value it read, as its log shows it,
3
+ # and whether the value was fine
4
+ #
5
+ # @!attribute value
6
+ # Return the value as the history shows it
7
+ #
8
+ # @return [#to_s] the formatted value
9
+ #
10
+ # @api private
11
+ #
12
+ # @!attribute critical
13
+ # Return whether the value was fine
14
+ #
15
+ # It's true for a value that passed the check, and false for one that
16
+ # counts toward the watch firing.
17
+ #
18
+ # @return [Boolean] whether the value passed the check
19
+ #
20
+ # @api private
21
+ #
22
+ # @api private
2
23
  HistoryValue = Struct.new(:value, :critical)
3
24
 
25
+ # Runs a process's check every so often, and fires its events when the
26
+ # check fails often enough
27
+ #
28
+ # A config's `checks :mem_usage, every: 10.seconds, below: 100.megabytes,
29
+ # times: [3, 5]` makes one: it runs every 10 seconds, and fires (restarts
30
+ # the process, by default) when 3 of the last 5 values failed.
31
+ #
32
+ # @api private
4
33
  class ConditionWatch
5
- attr_accessor :logger, :name
6
- EMPTY_ARRAY = [].freeze # no need to recreate one every tick
34
+ # Return the logger the watch writes its history to
35
+ #
36
+ # @return [Logger] the process's logger
37
+ #
38
+ # @api private
39
+ attr_accessor :logger
7
40
 
41
+ # Return the check's name, such as :mem_usage
42
+ #
43
+ # @return [Symbol] the name a config gave the check
44
+ #
45
+ # @api private
46
+ attr_accessor :name
47
+
48
+ no_events = [] #: Array[Symbol]
49
+
50
+ # The events of a run that fires nothing, made once instead of every tick
51
+ #
52
+ # @api private
53
+ EMPTY_ARRAY = no_events.freeze
54
+
55
+ # Create a watch of a check
56
+ #
57
+ # @param [Symbol] name the check's name, such as :mem_usage
58
+ # @param [Hash{Symbol => Object}] options the check's options. The
59
+ # watch takes :logger, :fires (the events to fire, [:restart] by
60
+ # default), :every (seconds between runs), :times (how many of the
61
+ # last values must fail, as a count or [count, of]), and
62
+ # :include_children. The check gets the rest.
63
+ #
64
+ # @return [void]
65
+ #
66
+ # @api private
8
67
  def initialize(name, options = {})
9
68
  @name = name
10
69
 
11
70
  @logger = options.delete(:logger)
12
71
  @fires = options.key?(:fires) ? Array(options.delete(:fires)) : [:restart]
13
72
  @every = options.delete(:every)
14
- @times = options.delete(:times) || [1, 1]
15
- @times = [@times, @times] unless @times.is_a?(Array) # handles times: 5
73
+ @times = Array(options.delete(:times) || 1) # handles times: 5
16
74
  @include_children = options.delete(:include_children) || false
17
75
 
18
76
  clear_history!
@@ -20,15 +78,35 @@ module Bluepill
20
78
  @process_condition = ProcessConditions[@name].new(options)
21
79
  end
22
80
 
23
- def run(pid, tick_number = Time.now.to_i)
24
- if @last_ran_at.nil? || (@last_ran_at + @every) <= tick_number
81
+ # Run the check if it's due, and return the events to fire
82
+ #
83
+ # A check is due on its first run, and then every `every` seconds. A
84
+ # run adds the value to the history and logs the history.
85
+ #
86
+ # @param [Integer, nil] pid the process's pid
87
+ # @param [Integer] tick_number the current time, in seconds
88
+ #
89
+ # @return [Array<Symbol>] the events to fire, empty unless the check
90
+ # ran and failed often enough
91
+ #
92
+ # @raise [StandardError] whatever the check raised, other than a
93
+ # System::PsError, after logging it
94
+ #
95
+ # @api private
96
+ def run(pid, tick_number = Integer(Time.now))
97
+ last_ran_at = @last_ran_at
98
+ if last_ran_at.nil? || (last_ran_at + @every) <= tick_number
25
99
  @last_ran_at = tick_number
26
100
 
27
101
  begin
28
102
  value = @process_condition.run(pid, @include_children)
29
- rescue => e
103
+ rescue System::PsError => e
104
+ # No data this time, so leave the history alone and check again next interval
105
+ logger.warning("#{@name}: #{e}")
106
+ return EMPTY_ARRAY
107
+ rescue StandardError => e
30
108
  logger.err(e.backtrace)
31
- raise e
109
+ raise
32
110
  end
33
111
 
34
112
  @history << HistoryValue.new(@process_condition.format_value(value), @process_condition.check(value))
@@ -39,14 +117,31 @@ module Bluepill
39
117
  EMPTY_ARRAY
40
118
  end
41
119
 
120
+ # Forget the values the check read, as when the process changes state
121
+ #
122
+ # @return [Util::RotationalArray<HistoryValue>] the new, empty history
123
+ #
124
+ # @api private
42
125
  def clear_history!
43
126
  @history = Util::RotationalArray.new(@times.last)
44
127
  end
45
128
 
129
+ # Return whether enough of the last values failed for the watch to fire
130
+ #
131
+ # @return [Boolean] true when the failures reach the times the check
132
+ # allows
133
+ #
134
+ # @api private
46
135
  def fired?
47
136
  @history.count { |v| !v.critical } >= @times.first
48
137
  end
49
138
 
139
+ # Return the check's name and history, as the log shows it
140
+ #
141
+ # @return [String] such as "mem_usage: [312MB, 401MB*]", where * marks a
142
+ # value that failed
143
+ #
144
+ # @api private
50
145
  def to_s
51
146
  data = @history.collect { |v| "#{v.value}#{'*' unless v.critical}" }.join(', ')
52
147
  "#{@name}: [#{data}]\n"