bluepill 0.1.5 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +57 -1
  3. data/CONTRIBUTING.md +2 -2
  4. data/README.md +105 -8
  5. data/bin/bluepill +25 -30
  6. data/bluepill.gemspec +21 -28
  7. data/lib/bluepill/application/lifecycle.rb +166 -0
  8. data/lib/bluepill/application/listener.rb +124 -0
  9. data/lib/bluepill/application/previous_daemon.rb +81 -0
  10. data/lib/bluepill/application.rb +59 -231
  11. data/lib/bluepill/base_directory.rb +93 -0
  12. data/lib/bluepill/condition_watch.rb +134 -56
  13. data/lib/bluepill/controller/directory.rb +59 -0
  14. data/lib/bluepill/controller.rb +99 -94
  15. data/lib/bluepill/daemonize.rb +234 -0
  16. data/lib/bluepill/dsl/app_proxy.rb +30 -6
  17. data/lib/bluepill/dsl/process_factory/children.rb +71 -0
  18. data/lib/bluepill/dsl/process_factory/registry.rb +69 -0
  19. data/lib/bluepill/dsl/process_factory.rb +81 -92
  20. data/lib/bluepill/dsl/process_proxy.rb +109 -21
  21. data/lib/bluepill/group.rb +187 -78
  22. data/lib/bluepill/logger.rb +139 -48
  23. data/lib/bluepill/pid_file.rb +23 -0
  24. data/lib/bluepill/process/attributes.rb +379 -0
  25. data/lib/bluepill/process/children.rb +70 -0
  26. data/lib/bluepill/process/pid.rb +154 -0
  27. data/lib/bluepill/process/state_machine.rb +196 -0
  28. data/lib/bluepill/process/transition.rb +45 -0
  29. data/lib/bluepill/process/transition_callbacks.rb +101 -0
  30. data/lib/bluepill/process/user_commands.rb +71 -0
  31. data/lib/bluepill/process/watches.rb +141 -0
  32. data/lib/bluepill/process.rb +56 -653
  33. data/lib/bluepill/process_commands/execution.rb +89 -0
  34. data/lib/bluepill/process_commands/hooks.rb +186 -0
  35. data/lib/bluepill/process_commands/stop_signals.rb +114 -0
  36. data/lib/bluepill/process_commands.rb +104 -172
  37. data/lib/bluepill/process_conditions/always_true.rb +1 -1
  38. data/lib/bluepill/process_conditions/below_threshold.rb +36 -0
  39. data/lib/bluepill/process_conditions/cpu_usage.rb +1 -25
  40. data/lib/bluepill/process_conditions/file_time.rb +4 -5
  41. data/lib/bluepill/process_conditions/http.rb +52 -24
  42. data/lib/bluepill/process_conditions/mem_usage.rb +9 -22
  43. data/lib/bluepill/process_conditions/process_condition.rb +34 -3
  44. data/lib/bluepill/process_conditions/running_time.rb +1 -25
  45. data/lib/bluepill/process_conditions/zombie_process.rb +1 -1
  46. data/lib/bluepill/process_conditions.rb +3 -2
  47. data/lib/bluepill/process_journal/cleanup.rb +200 -0
  48. data/lib/bluepill/process_journal/lock.rb +106 -0
  49. data/lib/bluepill/process_journal.rb +126 -233
  50. data/lib/bluepill/process_statistics.rb +2 -2
  51. data/lib/bluepill/socket.rb +60 -20
  52. data/lib/bluepill/system/command_setup.rb +98 -0
  53. data/lib/bluepill/system/commands.rb +219 -248
  54. data/lib/bluepill/system/daemon.rb +161 -0
  55. data/lib/bluepill/system/ps.rb +85 -0
  56. data/lib/bluepill/system/ps_table.rb +163 -0
  57. data/lib/bluepill/system.rb +45 -126
  58. data/lib/bluepill/trigger.rb +52 -26
  59. data/lib/bluepill/triggers/flapping.rb +7 -15
  60. data/lib/bluepill/units.rb +126 -0
  61. data/lib/bluepill/util/inflections.rb +34 -0
  62. data/lib/bluepill/util/levenshtein.rb +53 -0
  63. data/lib/bluepill/util/rotational_array.rb +1 -1
  64. data/lib/bluepill/version.rb +13 -12
  65. data/lib/bluepill.rb +33 -36
  66. data/sig/bluepill/application.rbs +257 -0
  67. data/sig/bluepill/checks.rbs +203 -0
  68. data/sig/bluepill/dsl.rbs +126 -0
  69. data/sig/bluepill/logger.rbs +104 -0
  70. data/sig/bluepill/process.rbs +542 -0
  71. data/sig/bluepill/system.rbs +316 -0
  72. data/sig/bluepill.rbs +47 -869
  73. metadata +39 -45
@@ -9,15 +9,30 @@ module Bluepill
9
9
  #
10
10
  # @api private
11
11
  class ProcessCondition
12
+ # Return the logger the condition can write to
13
+ #
14
+ # It's the watch's logger, so lines carry the app's and the process's
15
+ # names. A condition that calls super in its own initialize has it from
16
+ # then on.
17
+ #
18
+ # @example
19
+ # logger.info("free disk: #{free}")
20
+ #
21
+ # @return [Logger, nil] the process's logger
22
+ #
23
+ # @api public
24
+ attr_accessor :logger
25
+
12
26
  # Create a condition with the options a config gave its check
13
27
  #
14
28
  # @param [Hash{Symbol => Object}] options the check's options, such as
15
- # below: 100.megabytes
29
+ # below: 100.megabytes, and the watch's logger as logger:
16
30
  #
17
31
  # @return [void]
18
32
  #
19
33
  # @api private
20
34
  def initialize(options = {})
35
+ @logger = options.delete(:logger)
21
36
  @options = options
22
37
  end
23
38
 
@@ -32,11 +47,27 @@ module Bluepill
32
47
  #
33
48
  # @api private
34
49
  def run(_pid, _include_children)
35
- raise 'Implement in subclass!'
50
+ raise "Implement in subclass!"
36
51
  end
37
52
 
38
53
  # Return whether a value is fine
39
54
  #
55
+ # A subclass implements it. This one calls check, so a custom check
56
+ # written for bluepill 0.1, which defines check, still works.
57
+ #
58
+ # @param [Object] value what run read
59
+ #
60
+ # @return [Boolean] true when the value is fine, false when it's not
61
+ #
62
+ # @raise [RuntimeError] unless a subclass implements check? or check
63
+ #
64
+ # @api private
65
+ def check?(value)
66
+ check(value)
67
+ end
68
+
69
+ # Return whether a value is fine, as bluepill 0.1 named check?
70
+ #
40
71
  # @param [Object] _value what run read
41
72
  #
42
73
  # @return [Boolean] true when the value is fine, false when it's not
@@ -45,7 +76,7 @@ module Bluepill
45
76
  #
46
77
  # @api private
47
78
  def check(_value)
48
- raise 'Implement in subclass!'
79
+ raise "Implement in subclass!"
49
80
  end
50
81
 
51
82
  # Format a value for the watch's history and log
@@ -10,20 +10,7 @@ module Bluepill
10
10
  # process.checks :running_time, every: 10.minutes, below: 24.hours
11
11
  #
12
12
  # @api private
13
- class RunningTime < ProcessCondition
14
- # Create the check
15
- #
16
- # @param [Hash{Symbol => Object}] options the check's options, with
17
- # :below, the longest the process may run, in seconds
18
- #
19
- # @return [void]
20
- #
21
- # @api private
22
- def initialize(options = {})
23
- super
24
- @below = options[:below]
25
- end
26
-
13
+ class RunningTime < BelowThreshold
27
14
  # Read how long the process has run from the tick's ps table
28
15
  #
29
16
  # @param [Integer, nil] pid the process's pid
@@ -36,17 +23,6 @@ module Bluepill
36
23
  # Like the other ps conditions, a process missing from ps reads as 0
37
24
  System.running_time(pid).to_i
38
25
  end
39
-
40
- # Return whether the process has run for less than the limit
41
- #
42
- # @param [Integer] value the seconds run read
43
- #
44
- # @return [Boolean] true when it's below the limit
45
- #
46
- # @api private
47
- def check(value)
48
- value < @below
49
- end
50
26
  end
51
27
  end
52
28
  end
@@ -31,7 +31,7 @@ module Bluepill
31
31
  # @return [Boolean] true unless the command line says <defunct>
32
32
  #
33
33
  # @api private
34
- def check(value)
34
+ def check?(value)
35
35
  (value =~ /<defunct>/).nil?
36
36
  end
37
37
  end
@@ -15,12 +15,13 @@ module Bluepill
15
15
  #
16
16
  # @api private
17
17
  def self.[](name)
18
- const_get(name.to_s.camelcase)
18
+ const_get(Util.camelize(name))
19
19
  end
20
20
  end
21
21
  end
22
22
 
23
- require 'bluepill/process_conditions/process_condition'
23
+ require "bluepill/process_conditions/process_condition"
24
+ require "bluepill/process_conditions/below_threshold"
24
25
  Dir["#{File.dirname(__FILE__)}/process_conditions/*.rb"].sort.each do |pc|
25
26
  require pc
26
27
  end
@@ -0,0 +1,200 @@
1
+ module Bluepill
2
+ module ProcessJournal
3
+ # Kill what an earlier run left in a process's journals
4
+ #
5
+ # Before bluepill starts a process, it kills the processes and process
6
+ # groups its journals list that are still the ones bluepill journaled,
7
+ # and then clears the journals. ProcessJournal extends it.
8
+ #
9
+ # @api private
10
+ module Cleanup
11
+ # Kill the processes and process groups that a process's journals list
12
+ #
13
+ # @param [String] journal_name the process's name
14
+ #
15
+ # @return [void]
16
+ #
17
+ # @api private
18
+ def kill_all_from_journal(journal_name)
19
+ kill_all_pids_from_journal(journal_name)
20
+ kill_all_pgids_from_journal(journal_name)
21
+ end
22
+
23
+ # Kill the process groups that a process's journals list
24
+ #
25
+ # That's the app's journal, and the one that bluepill 0.1 and earlier
26
+ # shared between apps.
27
+ #
28
+ # @param [String] journal_name the process's name
29
+ #
30
+ # @return [void]
31
+ #
32
+ # @api private
33
+ def kill_all_pgids_from_journal(journal_name)
34
+ kill_all_ids_from_journal(PGID, journal_name)
35
+ end
36
+
37
+ # Kill the processes that a process's journals list
38
+ #
39
+ # That's the app's journal, and the one that bluepill 0.1 and earlier
40
+ # shared between apps.
41
+ #
42
+ # @param [String] journal_name the process's name
43
+ #
44
+ # @return [void]
45
+ #
46
+ # @api private
47
+ def kill_all_pids_from_journal(journal_name)
48
+ kill_all_ids_from_journal(PID, journal_name)
49
+ end
50
+
51
+ # Kill the ids that one kind of a process's journals list
52
+ #
53
+ # That's the app's journal, and the one that bluepill 0.1 and earlier
54
+ # shared between apps.
55
+ #
56
+ # @param [Kind] kind PID or PGID
57
+ # @param [String] journal_name the process's name
58
+ #
59
+ # @return [void]
60
+ #
61
+ # @api private
62
+ def kill_all_ids_from_journal(kind, journal_name)
63
+ journal_dirs.each do |dir|
64
+ filename = journal_filename(kind, journal_name, dir)
65
+ kill_all_from_journal_file(filename, journal_ids(kind, filename), kind)
66
+ end
67
+ end
68
+
69
+ # Return the directories a process's journals can be in
70
+ #
71
+ # @return [Array<String>] the app's, and then the one before 0.2
72
+ #
73
+ # @api private
74
+ def journal_dirs
75
+ [journal_base_dir, legacy_journal_dir]
76
+ end
77
+
78
+ # Kill the ids a journal lists, and then delete the journal
79
+ #
80
+ # Only kills the ids that still belong to what bluepill journaled (see
81
+ # journaled_ids). Sends TERM, waits a second for any survivors, then
82
+ # sends KILL to those still alive. The kind's sign makes process group
83
+ # ids target their groups.
84
+ #
85
+ # @param [String] filename the journal's path
86
+ # @param [Array<Integer>] ids the pids or process group ids in it
87
+ # @param [Kind] kind PID or PGID
88
+ #
89
+ # @return [void]
90
+ #
91
+ # @api private
92
+ def kill_all_from_journal_file(filename, ids, kind)
93
+ if ids.empty?
94
+ logger.debug("No previous process journal - Skipping cleanup")
95
+ return
96
+ end
97
+
98
+ acquire_atomic_fs_lock(filename) do
99
+ term_then_kill_journaled(journaled_ids(filename, ids, kind), kind)
100
+ System.delete_if_exists(filename) # reset journal
101
+ logger.debug("Journal cleanup completed")
102
+ end
103
+ end
104
+
105
+ # Send TERM to journaled ids, and KILL to any still alive a second later
106
+ #
107
+ # It doesn't wait when every id exited on TERM.
108
+ #
109
+ # @param [Array<Integer>] ids the pids or process group ids
110
+ # @param [Kind] kind PID or PGID
111
+ #
112
+ # @return [void]
113
+ #
114
+ # @api private
115
+ def term_then_kill_journaled(ids, kind)
116
+ signal_journaled("TERM", ids, kind)
117
+ survivors = ids.select { |id| System.pid_alive?(kind.sign * id) }
118
+ return if survivors.empty?
119
+
120
+ sleep(1)
121
+ survivors = survivors.select { |id| System.pid_alive?(kind.sign * id) }
122
+ signal_journaled("KILL", survivors, kind)
123
+ end
124
+
125
+ # Return the ids that still belong to what bluepill journaled
126
+ #
127
+ # See same_journaled_process? for how it tells.
128
+ #
129
+ # @param [String] filename the journal's path
130
+ # @param [Array<Integer>] ids the valid ids in it
131
+ # @param [Kind] kind PID or PGID
132
+ #
133
+ # @return [Array<Integer>] the ids that are safe to signal, or none if
134
+ # the journal was cleared since its ids were read
135
+ #
136
+ # @api private
137
+ def journaled_ids(filename, ids, kind)
138
+ # Journals only grow until they're cleared, under the lock this runs in
139
+ return [] unless File.exist?(filename)
140
+
141
+ started = Hash[journal_entries(filename)]
142
+ boot = System.boot_time
143
+ written = Integer(File.mtime(filename))
144
+ ids.select do |id|
145
+ ours = same_journaled_process?(id, started.fetch(id), boot, written)
146
+ logger.info("Skipping old #{kind.label} #{id}, which isn't the one bluepill started") unless ours
147
+ ours
148
+ end
149
+ end
150
+
151
+ # Return whether an id still belongs to the process bluepill journaled
152
+ #
153
+ # An id whose process is running is kept only if that process started
154
+ # when the journal says, give or take 2 seconds, since ps's elapsed time
155
+ # is only accurate to a second. One whose process has exited (a process
156
+ # group whose leader is gone) is kept only if it was journaled since the
157
+ # machine booted, and so is one without a start time.
158
+ #
159
+ # @param [Integer] id the pid or process group id
160
+ # @param [Integer, nil] journaled when the journal says its process
161
+ # started, or nil for a line from bluepill 0.1 and earlier
162
+ # @param [Integer, nil] boot when the machine booted, or nil if ps
163
+ # can't tell
164
+ # @param [Integer] written when the journal was last written
165
+ #
166
+ # @return [Boolean] whether it's safe to signal
167
+ #
168
+ # @api private
169
+ def same_journaled_process?(id, journaled, boot, written)
170
+ now = System.started_at(id)
171
+ if now && journaled
172
+ (now - journaled).abs <= 2
173
+ else
174
+ !boot.nil? && (journaled || written) >= boot - 2
175
+ end
176
+ end
177
+
178
+ # Send a signal to each journaled id, and log what happened
179
+ #
180
+ # @param [String] signal TERM or KILL
181
+ # @param [Array<Integer>] ids the pids or process group ids
182
+ # @param [Kind] kind PID or PGID
183
+ #
184
+ # @return [void]
185
+ #
186
+ # @api private
187
+ def signal_journaled(signal, ids, kind)
188
+ past, verb = signal.eql?("TERM") ? %w[Termed term] : %w[Killed kill]
189
+ ids.each do |id|
190
+ begin
191
+ ::Process.kill(signal, kind.sign * id)
192
+ logger.info("#{past} old #{kind.label} #{id}")
193
+ rescue Errno::ESRCH
194
+ logger.debug("Unable to #{verb} missing #{kind.label} #{id}")
195
+ end
196
+ end
197
+ end
198
+ end
199
+ end
200
+ end
@@ -0,0 +1,106 @@
1
+ module Bluepill
2
+ module ProcessJournal
3
+ # Lock a journal while bluepill reads and writes it
4
+ #
5
+ # Processes that run as other users write to the same journals, so each
6
+ # journal has a lock: a directory next to it, since making one is atomic
7
+ # on POSIX filesystems and f.flock(File::LOCK_SH) is not available on all
8
+ # platforms. ProcessJournal extends it.
9
+ #
10
+ # @api private
11
+ module Lock
12
+ # Run a block while holding a lock on a journal
13
+ #
14
+ # It waits up to 10 seconds for another process's lock.
15
+ #
16
+ # @param [String] name the journal's file name
17
+ #
18
+ # @yield while holding the lock
19
+ # @yieldreturn [Object] what the method returns
20
+ #
21
+ # @return [Object] what the block returns
22
+ #
23
+ # @raise [RuntimeError] if the lock is still held after 10 tries
24
+ #
25
+ # @api private
26
+ def acquire_atomic_fs_lock(name)
27
+ lock = "#{name}.lock"
28
+ make_atomic_fs_lock(lock)
29
+ logger.debug("Acquired lock #{lock}")
30
+ begin
31
+ yield
32
+ ensure
33
+ clear_atomic_fs_lock(lock)
34
+ end
35
+ end
36
+
37
+ # Remove the app's journal locks, which a daemon that died left behind
38
+ #
39
+ # Without this, a stale lock would make the next write to its journal
40
+ # wait 10 seconds and fail.
41
+ #
42
+ # @return [void]
43
+ #
44
+ # @api private
45
+ def clear_all_atomic_fs_locks
46
+ Dir[File.join(journal_base_dir, ".*.lock")].each do |lock|
47
+ clear_atomic_fs_lock(lock)
48
+ end
49
+ end
50
+
51
+ # Release a journal lock, if it's there
52
+ #
53
+ # @param [String] name the lock's directory
54
+ #
55
+ # @return [void]
56
+ #
57
+ # @api private
58
+ def clear_atomic_fs_lock(name)
59
+ return unless File.directory?(name)
60
+
61
+ Dir.rmdir(name)
62
+ logger.debug("Cleared lock #{name}")
63
+ end
64
+
65
+ # Make a lock's directory, waiting for another process to release it
66
+ #
67
+ # @param [String] lock the lock's directory
68
+ #
69
+ # @return [void]
70
+ #
71
+ # @raise [RuntimeError] if the lock is still held after 10 tries
72
+ #
73
+ # @api private
74
+ def make_atomic_fs_lock(lock)
75
+ attempts = 0
76
+ begin
77
+ Dir.mkdir lock, 0o700
78
+ rescue Errno::EEXIST
79
+ attempts += 1
80
+ wait_for_atomic_fs_lock(lock, attempts)
81
+ retry
82
+ end
83
+ end
84
+
85
+ # Wait a second for another process's lock, unless that's been 10 tries
86
+ #
87
+ # @param [String] lock the lock's directory
88
+ # @param [Integer] attempts how many times making it has failed
89
+ #
90
+ # @return [void]
91
+ #
92
+ # @raise [RuntimeError] after the 10th try
93
+ #
94
+ # @api private
95
+ def wait_for_atomic_fs_lock(lock, attempts)
96
+ if attempts < 10
97
+ logger.debug("Waiting for lock #{lock}")
98
+ sleep 1
99
+ else
100
+ logger.info("Timeout waiting for lock #{lock}")
101
+ raise "Timeout waiting for lock #{lock}"
102
+ end
103
+ end
104
+ end
105
+ end
106
+ end