bluepill 0.2.7 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +144 -1
- data/README.md +17 -10
- data/bin/bluepill +8 -2
- data/bluepill.gemspec +1 -1
- data/lib/bluepill/application/lifecycle.rb +43 -31
- data/lib/bluepill/application/listener.rb +1 -1
- data/lib/bluepill/application/previous_daemon.rb +14 -14
- data/lib/bluepill/application.rb +48 -28
- data/lib/bluepill/base_directory.rb +18 -20
- data/lib/bluepill/check_registry.rb +104 -0
- data/lib/bluepill/clock.rb +1 -3
- data/lib/bluepill/condition_watch.rb +61 -67
- data/lib/bluepill/controller/directory.rb +1 -3
- data/lib/bluepill/controller.rb +46 -38
- data/lib/bluepill/dsl/app_proxy.rb +69 -58
- data/lib/bluepill/dsl/attributes.rb +135 -0
- data/lib/bluepill/dsl/process_factory/child_factory.rb +80 -0
- data/lib/bluepill/dsl/process_factory/registry.rb +6 -26
- data/lib/bluepill/dsl/process_factory.rb +86 -70
- data/lib/bluepill/dsl/process_proxy.rb +61 -158
- data/lib/bluepill/dsl.rb +15 -6
- data/lib/bluepill/errors.rb +190 -0
- data/lib/bluepill/group.rb +14 -37
- data/lib/bluepill/logger.rb +55 -97
- data/lib/bluepill/process/attributes.rb +66 -32
- data/lib/bluepill/process/children.rb +4 -6
- data/lib/bluepill/process/pid.rb +41 -35
- data/lib/bluepill/process/state_machine.rb +15 -59
- data/lib/bluepill/process/stop_signals.rb +120 -0
- data/lib/bluepill/process/transition.rb +1 -3
- data/lib/bluepill/process/transition_callbacks.rb +2 -19
- data/lib/bluepill/process/triggers.rb +90 -0
- data/lib/bluepill/process/user_commands.rb +1 -1
- data/lib/bluepill/process/watches.rb +9 -23
- data/lib/bluepill/process.rb +70 -55
- data/lib/bluepill/process_commands/execution.rb +14 -22
- data/lib/bluepill/process_commands/hooks.rb +1 -31
- data/lib/bluepill/process_commands/stop_signals.rb +12 -17
- data/lib/bluepill/process_commands.rb +10 -8
- data/lib/bluepill/process_conditions/always_true.rb +2 -6
- data/lib/bluepill/process_conditions/below_threshold.rb +6 -8
- data/lib/bluepill/process_conditions/cpu_usage.rb +1 -4
- data/lib/bluepill/process_conditions/file_time.rb +8 -9
- data/lib/bluepill/process_conditions/http.rb +28 -43
- data/lib/bluepill/process_conditions/mem_usage.rb +7 -45
- data/lib/bluepill/process_conditions/process_condition.rb +30 -32
- data/lib/bluepill/process_conditions/zombie_process.rb +2 -6
- data/lib/bluepill/process_conditions.rb +5 -14
- data/lib/bluepill/process_journal/cleanup.rb +5 -53
- data/lib/bluepill/process_journal/config.rb +73 -0
- data/lib/bluepill/process_journal/entries.rb +128 -0
- data/lib/bluepill/process_journal/journal.rb +69 -0
- data/lib/bluepill/process_journal/kind.rb +60 -0
- data/lib/bluepill/process_journal/lock.rb +9 -4
- data/lib/bluepill/process_journal.rb +13 -324
- data/lib/bluepill/process_statistics.rb +3 -7
- data/lib/bluepill/socket.rb +19 -17
- data/lib/bluepill/system/command_options.rb +135 -0
- data/lib/bluepill/system/command_result.rb +73 -0
- data/lib/bluepill/system/command_setup.rb +3 -4
- data/lib/bluepill/system/commands.rb +21 -18
- data/lib/bluepill/system/daemon.rb +19 -14
- data/lib/bluepill/system/files.rb +35 -0
- data/lib/bluepill/system/ps.rb +20 -17
- data/lib/bluepill/system/ps_table.rb +7 -9
- data/lib/bluepill/system/usage.rb +155 -0
- data/lib/bluepill/system.rb +7 -192
- data/lib/bluepill/trigger.rb +22 -32
- data/lib/bluepill/triggers/flapping.rb +12 -8
- data/lib/bluepill/units.rb +7 -21
- data/lib/bluepill/util/log_error.rb +28 -0
- data/lib/bluepill/version.rb +12 -22
- data/lib/bluepill.rb +1 -0
- data/sig/bluepill/application.rbs +17 -12
- data/sig/bluepill/attributes.rbs +48 -0
- data/sig/bluepill/checks.rbs +43 -30
- data/sig/bluepill/dsl.rbs +98 -55
- data/sig/bluepill/errors.rbs +48 -0
- data/sig/bluepill/logger.rbs +15 -31
- data/sig/bluepill/process.rbs +42 -304
- data/sig/bluepill/process_commands.rbs +235 -0
- data/sig/bluepill/stop_signals.rbs +21 -0
- data/sig/bluepill/system.rbs +154 -99
- data/sig/bluepill/version.rbs +17 -0
- data/sig/bluepill.rbs +4 -17
- data/sig/manifest.yaml +1 -0
- metadata +22 -3
- data/lib/bluepill/dsl/process_factory/children.rb +0 -73
|
@@ -2,6 +2,10 @@
|
|
|
2
2
|
|
|
3
3
|
module Bluepill
|
|
4
4
|
module System
|
|
5
|
+
# No pids, which get_children returns for a process with no children
|
|
6
|
+
no_pids = [] #: Array[Integer]
|
|
7
|
+
NO_PIDS = no_pids.freeze
|
|
8
|
+
|
|
5
9
|
# A tick's table of processes, as ps listed them
|
|
6
10
|
#
|
|
7
11
|
# Each row is a process's ps fields, Strings in the order of the column
|
|
@@ -76,9 +80,7 @@ module Bluepill
|
|
|
76
80
|
# @return [undefined]
|
|
77
81
|
#
|
|
78
82
|
# @api private
|
|
79
|
-
def initialize(rows)
|
|
80
|
-
@rows = rows
|
|
81
|
-
end
|
|
83
|
+
def initialize(rows) = @rows = rows
|
|
82
84
|
|
|
83
85
|
# Return a process's ps fields
|
|
84
86
|
#
|
|
@@ -87,9 +89,7 @@ module Bluepill
|
|
|
87
89
|
# @return [Array<String>, nil] its fields, or nil if ps doesn't list it
|
|
88
90
|
#
|
|
89
91
|
# @api private
|
|
90
|
-
def row(pid)
|
|
91
|
-
@rows[pid]
|
|
92
|
-
end
|
|
92
|
+
def row(pid) = @rows[pid]
|
|
93
93
|
|
|
94
94
|
# Return the pids of a process's children
|
|
95
95
|
#
|
|
@@ -99,9 +99,7 @@ module Bluepill
|
|
|
99
99
|
# NO_PIDS if it has none
|
|
100
100
|
#
|
|
101
101
|
# @api private
|
|
102
|
-
def children(pid)
|
|
103
|
-
children_by_parent.fetch(pid, NO_PIDS)
|
|
104
|
-
end
|
|
102
|
+
def children(pid) = children_by_parent.fetch(pid, NO_PIDS)
|
|
105
103
|
|
|
106
104
|
# Return the pids of a process's descendants
|
|
107
105
|
#
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bluepill
|
|
4
|
+
module System
|
|
5
|
+
# Ask the system about a process: whether it's alive, when it started,
|
|
6
|
+
# and what this tick's ps table says it uses
|
|
7
|
+
#
|
|
8
|
+
# System extends this module, so these are System's module functions.
|
|
9
|
+
#
|
|
10
|
+
# @api private
|
|
11
|
+
module Usage
|
|
12
|
+
# Return whether a process is running
|
|
13
|
+
#
|
|
14
|
+
# @param [Integer] pid the process's pid
|
|
15
|
+
#
|
|
16
|
+
# @return [Boolean] true if the process exists, even if bluepill may
|
|
17
|
+
# not signal it
|
|
18
|
+
#
|
|
19
|
+
# @api private
|
|
20
|
+
def pid_alive?(pid)
|
|
21
|
+
::Process.kill(0, pid)
|
|
22
|
+
true
|
|
23
|
+
rescue Errno::EPERM # no permission, but it is definitely alive
|
|
24
|
+
true
|
|
25
|
+
rescue
|
|
26
|
+
false # no such process (ESRCH), or no pid
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Return when a process started, in seconds since the epoch
|
|
30
|
+
#
|
|
31
|
+
# It's read from ps's elapsed time for the process, so it's accurate to
|
|
32
|
+
# about a second, and works wherever ps does.
|
|
33
|
+
#
|
|
34
|
+
# @param [Integer] pid the process's pid
|
|
35
|
+
#
|
|
36
|
+
# @return [Integer, nil] when it started, or nil if there's no such
|
|
37
|
+
# process
|
|
38
|
+
#
|
|
39
|
+
# @api private
|
|
40
|
+
def started_at(pid)
|
|
41
|
+
# ps says why on stderr when there's no such process
|
|
42
|
+
elapsed = IO.popen(["ps", "-o", "etime=", "-p", Integer(pid).to_s], err: File::NULL, &:read).rstrip
|
|
43
|
+
Integer(Time.now) - parse_elapsed_time(elapsed) unless elapsed.empty?
|
|
44
|
+
rescue SystemCallError
|
|
45
|
+
nil
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
# Return when the machine booted, in seconds since the epoch
|
|
49
|
+
#
|
|
50
|
+
# That's when its first process, pid 1, started. In a container, it's
|
|
51
|
+
# when the container started, which is when its pids start over.
|
|
52
|
+
#
|
|
53
|
+
# @return [Integer, nil] when it booted, or nil if ps can't tell
|
|
54
|
+
#
|
|
55
|
+
# @api private
|
|
56
|
+
def boot_time = started_at(1)
|
|
57
|
+
|
|
58
|
+
# Return a process's CPU usage, from this tick's ps table
|
|
59
|
+
#
|
|
60
|
+
# @param [Integer, nil] pid the process's pid
|
|
61
|
+
# @param [Boolean] include_children whether to add its descendants'
|
|
62
|
+
#
|
|
63
|
+
# @return [Float, nil] the percent of a CPU it uses, or nil if ps doesn't
|
|
64
|
+
# list it
|
|
65
|
+
#
|
|
66
|
+
# @raise [PsError] if ps could not be run this tick
|
|
67
|
+
#
|
|
68
|
+
# @api private
|
|
69
|
+
def cpu_usage(pid, include_children) = usage(pid, include_children, PsTable::PCPU)
|
|
70
|
+
|
|
71
|
+
# Return a process's resident memory, from this tick's ps table
|
|
72
|
+
#
|
|
73
|
+
# @param [Integer, nil] pid the process's pid
|
|
74
|
+
# @param [Boolean] include_children whether to add its descendants'
|
|
75
|
+
#
|
|
76
|
+
# @return [Float, nil] its resident memory in kilobytes, or nil if ps
|
|
77
|
+
# doesn't list it
|
|
78
|
+
#
|
|
79
|
+
# @raise [PsError] if ps could not be run this tick
|
|
80
|
+
#
|
|
81
|
+
# @api private
|
|
82
|
+
def memory_usage(pid, include_children) = usage(pid, include_children, PsTable::RSS)
|
|
83
|
+
|
|
84
|
+
# Return a column of a process's row, from this tick's ps table, as a Float
|
|
85
|
+
#
|
|
86
|
+
# @param [Integer, nil] pid the process's pid
|
|
87
|
+
# @param [Boolean] include_children whether to add its descendants'
|
|
88
|
+
# @param [Integer] column the column, PsTable::PCPU or PsTable::RSS
|
|
89
|
+
#
|
|
90
|
+
# @return [Float, nil] the process's value, and its descendants' if
|
|
91
|
+
# asked for, or nil if ps doesn't list it
|
|
92
|
+
#
|
|
93
|
+
# @raise [PsError] if ps could not be run this tick
|
|
94
|
+
#
|
|
95
|
+
# @api private
|
|
96
|
+
def usage(pid, include_children, column)
|
|
97
|
+
ps = ps_axu
|
|
98
|
+
row = ps.row(pid)
|
|
99
|
+
return unless row
|
|
100
|
+
|
|
101
|
+
used = row.fetch(column).to_f
|
|
102
|
+
include_children ? ps.sum(get_children(pid), column, used) : used
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# Return how long a process has been running, from this tick's ps table
|
|
106
|
+
#
|
|
107
|
+
# @param [Integer, nil] pid the process's pid
|
|
108
|
+
#
|
|
109
|
+
# @return [Integer, nil] its running time in seconds, or nil if ps
|
|
110
|
+
# doesn't list it
|
|
111
|
+
#
|
|
112
|
+
# @raise [PsError] if ps could not be run this tick
|
|
113
|
+
#
|
|
114
|
+
# @api private
|
|
115
|
+
def running_time(pid)
|
|
116
|
+
row = ps_axu.row(pid)
|
|
117
|
+
parse_elapsed_time(row.fetch(PsTable::ETIME)) if row
|
|
118
|
+
end
|
|
119
|
+
|
|
120
|
+
# Return a process's command line, from this tick's ps table
|
|
121
|
+
#
|
|
122
|
+
# @param [Integer, nil] pid the process's pid
|
|
123
|
+
#
|
|
124
|
+
# @return [String, nil] its command and arguments, or nil if ps doesn't
|
|
125
|
+
# list it
|
|
126
|
+
#
|
|
127
|
+
# @raise [PsError] if ps could not be run this tick
|
|
128
|
+
#
|
|
129
|
+
# @api private
|
|
130
|
+
def command(pid)
|
|
131
|
+
row = ps_axu.row(pid)
|
|
132
|
+
row&.fetch(PsTable::COMMAND)
|
|
133
|
+
end
|
|
134
|
+
|
|
135
|
+
# Convert ps's elapsed time to seconds
|
|
136
|
+
#
|
|
137
|
+
# @param [String, nil] str an elapsed time, [[dd-]hh:]mm:ss
|
|
138
|
+
#
|
|
139
|
+
# @return [Integer] the seconds, or 0 if str isn't an elapsed time
|
|
140
|
+
#
|
|
141
|
+
# @api private
|
|
142
|
+
def parse_elapsed_time(str)
|
|
143
|
+
# [[dd-]hh:]mm:ss
|
|
144
|
+
match = /(?:(?:(\d+)-)?(\d\d):)?(\d\d):(\d\d)/.match(str)
|
|
145
|
+
if match
|
|
146
|
+
hours = (match[1].to_i * 24) + match[2].to_i
|
|
147
|
+
mins = (hours * 60) + match[3].to_i
|
|
148
|
+
(mins * 60) + match[4].to_i
|
|
149
|
+
else
|
|
150
|
+
0
|
|
151
|
+
end
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
end
|
|
155
|
+
end
|
data/lib/bluepill/system.rb
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
3
|
require "bluepill/system/ps"
|
|
4
|
+
require "bluepill/system/usage"
|
|
5
|
+
require "bluepill/system/files"
|
|
4
6
|
require "bluepill/system/commands"
|
|
5
7
|
|
|
6
8
|
module Bluepill
|
|
@@ -9,203 +11,16 @@ module Bluepill
|
|
|
9
11
|
# This represents the system that bluepill is running on. It memoizes
|
|
10
12
|
# the results of running ps each tick, so that every watch of every
|
|
11
13
|
# process doesn't fork a ps of its own. Running ps and reading its table
|
|
12
|
-
# is in System::Ps,
|
|
13
|
-
# System::
|
|
14
|
+
# is in System::Ps, asking about a process is in System::Usage, deleting
|
|
15
|
+
# files is in System::Files, and running commands is in System::Commands
|
|
16
|
+
# and System::Daemon. System only extends them.
|
|
14
17
|
#
|
|
15
18
|
# @api private
|
|
16
19
|
module System
|
|
17
20
|
extend Ps
|
|
21
|
+
extend Usage
|
|
22
|
+
extend Files
|
|
18
23
|
extend Commands
|
|
19
24
|
extend Daemon
|
|
20
|
-
|
|
21
|
-
# The mode that a daemon's stdout and stderr files are opened in
|
|
22
|
-
#
|
|
23
|
-
# @api private
|
|
24
|
-
APPEND_MODE = "a"
|
|
25
|
-
|
|
26
|
-
# Raised when ps could not be run, so there is no data for this tick
|
|
27
|
-
#
|
|
28
|
-
# @api private
|
|
29
|
-
class PsError < StandardError; end
|
|
30
|
-
|
|
31
|
-
# Return whether a process is running
|
|
32
|
-
#
|
|
33
|
-
# @param [Integer] pid the process's pid
|
|
34
|
-
#
|
|
35
|
-
# @return [Boolean] true if the process exists, even if bluepill may
|
|
36
|
-
# not signal it
|
|
37
|
-
#
|
|
38
|
-
# @api private
|
|
39
|
-
def self.pid_alive?(pid)
|
|
40
|
-
::Process.kill(0, pid)
|
|
41
|
-
true
|
|
42
|
-
rescue Errno::EPERM # no permission, but it is definitely alive
|
|
43
|
-
true
|
|
44
|
-
rescue
|
|
45
|
-
false # no such process (ESRCH), or no pid
|
|
46
|
-
end
|
|
47
|
-
|
|
48
|
-
# Return when a process started, in seconds since the epoch
|
|
49
|
-
#
|
|
50
|
-
# It's read from ps's elapsed time for the process, so it's accurate to
|
|
51
|
-
# about a second, and works wherever ps does.
|
|
52
|
-
#
|
|
53
|
-
# @param [Integer] pid the process's pid
|
|
54
|
-
#
|
|
55
|
-
# @return [Integer, nil] when it started, or nil if there's no such
|
|
56
|
-
# process
|
|
57
|
-
#
|
|
58
|
-
# @api private
|
|
59
|
-
def self.started_at(pid)
|
|
60
|
-
# ps says why on stderr when there's no such process
|
|
61
|
-
elapsed = IO.popen(["ps", "-o", "etime=", "-p", Integer(pid).to_s], err: File::NULL, &:read).rstrip
|
|
62
|
-
Integer(Time.now) - parse_elapsed_time(elapsed) unless elapsed.empty?
|
|
63
|
-
rescue SystemCallError
|
|
64
|
-
nil
|
|
65
|
-
end
|
|
66
|
-
|
|
67
|
-
# Return when the machine booted, in seconds since the epoch
|
|
68
|
-
#
|
|
69
|
-
# That's when its first process, pid 1, started. In a container, it's
|
|
70
|
-
# when the container started, which is when its pids start over.
|
|
71
|
-
#
|
|
72
|
-
# @return [Integer, nil] when it booted, or nil if ps can't tell
|
|
73
|
-
#
|
|
74
|
-
# @api private
|
|
75
|
-
def self.boot_time
|
|
76
|
-
started_at(1)
|
|
77
|
-
end
|
|
78
|
-
|
|
79
|
-
# Return a process's CPU usage, from this tick's ps table
|
|
80
|
-
#
|
|
81
|
-
# @param [Integer, nil] pid the process's pid
|
|
82
|
-
# @param [Boolean] include_children whether to add its descendants'
|
|
83
|
-
#
|
|
84
|
-
# @return [Float, nil] the percent of a CPU it uses, or nil if ps doesn't
|
|
85
|
-
# list it
|
|
86
|
-
#
|
|
87
|
-
# @raise [PsError] if ps could not be run this tick
|
|
88
|
-
#
|
|
89
|
-
# @api private
|
|
90
|
-
def self.cpu_usage(pid, include_children)
|
|
91
|
-
usage(pid, include_children, PsTable::PCPU)
|
|
92
|
-
end
|
|
93
|
-
|
|
94
|
-
# Return a process's resident memory, from this tick's ps table
|
|
95
|
-
#
|
|
96
|
-
# @param [Integer, nil] pid the process's pid
|
|
97
|
-
# @param [Boolean] include_children whether to add its descendants'
|
|
98
|
-
#
|
|
99
|
-
# @return [Float, nil] its resident memory in kilobytes, or nil if ps
|
|
100
|
-
# doesn't list it
|
|
101
|
-
#
|
|
102
|
-
# @raise [PsError] if ps could not be run this tick
|
|
103
|
-
#
|
|
104
|
-
# @api private
|
|
105
|
-
def self.memory_usage(pid, include_children)
|
|
106
|
-
usage(pid, include_children, PsTable::RSS)
|
|
107
|
-
end
|
|
108
|
-
|
|
109
|
-
# Return a column of a process's row, from this tick's ps table, as a Float
|
|
110
|
-
#
|
|
111
|
-
# @param [Integer, nil] pid the process's pid
|
|
112
|
-
# @param [Boolean] include_children whether to add its descendants'
|
|
113
|
-
# @param [Integer] column the column, PsTable::PCPU or PsTable::RSS
|
|
114
|
-
#
|
|
115
|
-
# @return [Float, nil] the process's value, and its descendants' if
|
|
116
|
-
# asked for, or nil if ps doesn't list it
|
|
117
|
-
#
|
|
118
|
-
# @raise [PsError] if ps could not be run this tick
|
|
119
|
-
#
|
|
120
|
-
# @api private
|
|
121
|
-
def self.usage(pid, include_children, column)
|
|
122
|
-
ps = ps_axu
|
|
123
|
-
row = ps.row(pid)
|
|
124
|
-
return unless row
|
|
125
|
-
|
|
126
|
-
used = row.fetch(column).to_f
|
|
127
|
-
include_children ? ps.sum(get_children(pid), column, used) : used
|
|
128
|
-
end
|
|
129
|
-
|
|
130
|
-
# Return how long a process has been running, from this tick's ps table
|
|
131
|
-
#
|
|
132
|
-
# @param [Integer, nil] pid the process's pid
|
|
133
|
-
#
|
|
134
|
-
# @return [Integer, nil] its running time in seconds, or nil if ps
|
|
135
|
-
# doesn't list it
|
|
136
|
-
#
|
|
137
|
-
# @raise [PsError] if ps could not be run this tick
|
|
138
|
-
#
|
|
139
|
-
# @api private
|
|
140
|
-
def self.running_time(pid)
|
|
141
|
-
row = ps_axu.row(pid)
|
|
142
|
-
parse_elapsed_time(row.fetch(PsTable::ETIME)) if row
|
|
143
|
-
end
|
|
144
|
-
|
|
145
|
-
# Return a process's command line, from this tick's ps table
|
|
146
|
-
#
|
|
147
|
-
# @param [Integer, nil] pid the process's pid
|
|
148
|
-
#
|
|
149
|
-
# @return [String, nil] its command and arguments, or nil if ps doesn't
|
|
150
|
-
# list it
|
|
151
|
-
#
|
|
152
|
-
# @raise [PsError] if ps could not be run this tick
|
|
153
|
-
#
|
|
154
|
-
# @api private
|
|
155
|
-
def self.command(pid)
|
|
156
|
-
row = ps_axu.row(pid)
|
|
157
|
-
row&.fetch(PsTable::COMMAND)
|
|
158
|
-
end
|
|
159
|
-
|
|
160
|
-
# Delete a file if it exists
|
|
161
|
-
#
|
|
162
|
-
# It tries three times when permission is denied, and then prints a
|
|
163
|
-
# warning instead of raising.
|
|
164
|
-
#
|
|
165
|
-
# @param [String, nil] filename the file's path
|
|
166
|
-
#
|
|
167
|
-
# @return [void]
|
|
168
|
-
#
|
|
169
|
-
# @api private
|
|
170
|
-
def self.delete_if_exists(filename)
|
|
171
|
-
tries = 0
|
|
172
|
-
|
|
173
|
-
begin
|
|
174
|
-
File.unlink(filename) if filename && File.exist?(filename)
|
|
175
|
-
rescue IOError, Errno::ENOENT
|
|
176
|
-
# Already gone, or another process removed it in between
|
|
177
|
-
rescue Errno::EACCES
|
|
178
|
-
retry if (tries += 1) < 3
|
|
179
|
-
$stderr.puts("Warning: permission denied trying to delete #{filename}")
|
|
180
|
-
end
|
|
181
|
-
end
|
|
182
|
-
|
|
183
|
-
# Watches run in their own threads, so only let one of them run ps per tick
|
|
184
|
-
#
|
|
185
|
-
# @api private
|
|
186
|
-
PS_MUTEX = Mutex.new
|
|
187
|
-
|
|
188
|
-
# No pids, which get_children returns for a process with no children
|
|
189
|
-
no_pids = [] #: Array[Integer]
|
|
190
|
-
NO_PIDS = no_pids.freeze
|
|
191
|
-
|
|
192
|
-
# Convert ps's elapsed time to seconds
|
|
193
|
-
#
|
|
194
|
-
# @param [String, nil] str an elapsed time, [[dd-]hh:]mm:ss
|
|
195
|
-
#
|
|
196
|
-
# @return [Integer] the seconds, or 0 if str isn't an elapsed time
|
|
197
|
-
#
|
|
198
|
-
# @api private
|
|
199
|
-
def self.parse_elapsed_time(str)
|
|
200
|
-
# [[dd-]hh:]mm:ss
|
|
201
|
-
match = /(?:(?:(\d+)-)?(\d\d):)?(\d\d):(\d\d)/.match(str)
|
|
202
|
-
if match
|
|
203
|
-
hours = (match[1].to_i * 24) + match[2].to_i
|
|
204
|
-
mins = (hours * 60) + match[3].to_i
|
|
205
|
-
(mins * 60) + match[4].to_i
|
|
206
|
-
else
|
|
207
|
-
0
|
|
208
|
-
end
|
|
209
|
-
end
|
|
210
25
|
end
|
|
211
26
|
end
|
data/lib/bluepill/trigger.rb
CHANGED
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
# frozen_string_literal: true
|
|
2
2
|
|
|
3
|
+
require "bluepill/check_registry"
|
|
4
|
+
require "bluepill/util/log_error"
|
|
5
|
+
|
|
3
6
|
module Bluepill
|
|
4
7
|
# Reacts to a process's state transitions, as one of a process's checks
|
|
5
8
|
#
|
|
6
9
|
# A config names a trigger the way it names a check, by its underscored
|
|
7
10
|
# class name: `checks :flapping` makes a Triggers::Flapping. Subclasses
|
|
8
|
-
#
|
|
11
|
+
# add themselves to CheckRegistry by that name, and implement notify.
|
|
9
12
|
#
|
|
10
13
|
# @api private
|
|
11
14
|
class Trigger
|
|
@@ -15,9 +18,14 @@ module Bluepill
|
|
|
15
18
|
# @api private
|
|
16
19
|
HALT = :halt
|
|
17
20
|
|
|
18
|
-
|
|
21
|
+
# The methods a subclass must define
|
|
22
|
+
#
|
|
23
|
+
# @api private
|
|
24
|
+
REQUIRED_METHODS = %i[notify].freeze
|
|
19
25
|
|
|
20
|
-
#
|
|
26
|
+
# Add a subclass to the checks a config can name
|
|
27
|
+
#
|
|
28
|
+
# A config names it by its underscored name, such as :flapping.
|
|
21
29
|
#
|
|
22
30
|
# @param [Class] klass the subclass
|
|
23
31
|
#
|
|
@@ -26,19 +34,7 @@ module Bluepill
|
|
|
26
34
|
# @api private
|
|
27
35
|
def self.inherited(klass)
|
|
28
36
|
super
|
|
29
|
-
|
|
30
|
-
end
|
|
31
|
-
|
|
32
|
-
# Return the trigger class a config names
|
|
33
|
-
#
|
|
34
|
-
# @param [Symbol] name the trigger's underscored name, such as :flapping
|
|
35
|
-
#
|
|
36
|
-
# @return [Class, nil] the Trigger subclass, or nil when no trigger has
|
|
37
|
-
# that name (so the name is a check's)
|
|
38
|
-
#
|
|
39
|
-
# @api private
|
|
40
|
-
def self.[](name)
|
|
41
|
-
@implementations[name]
|
|
37
|
+
CheckRegistry.add(klass, Trigger, REQUIRED_METHODS)
|
|
42
38
|
end
|
|
43
39
|
|
|
44
40
|
# Return the process the trigger watches
|
|
@@ -53,7 +49,7 @@ module Bluepill
|
|
|
53
49
|
# @return [Logger] the process's logger
|
|
54
50
|
#
|
|
55
51
|
# @api private
|
|
56
|
-
|
|
52
|
+
attr_reader :logger
|
|
57
53
|
|
|
58
54
|
# Return the events the trigger scheduled, each with its thread
|
|
59
55
|
#
|
|
@@ -69,15 +65,16 @@ module Bluepill
|
|
|
69
65
|
# Create a trigger for a process
|
|
70
66
|
#
|
|
71
67
|
# @param [Process] process the process to watch
|
|
72
|
-
# @param [
|
|
73
|
-
# :logger
|
|
68
|
+
# @param [Logger, nil] logger the process's logger
|
|
74
69
|
#
|
|
75
70
|
# @return [void]
|
|
76
71
|
#
|
|
72
|
+
# @raise [ArgumentError] for an option that no trigger takes
|
|
73
|
+
#
|
|
77
74
|
# @api private
|
|
78
|
-
def initialize(process,
|
|
75
|
+
def initialize(process, logger: nil)
|
|
79
76
|
self.process = process
|
|
80
|
-
|
|
77
|
+
@logger = logger
|
|
81
78
|
@mutex = Mutex.new
|
|
82
79
|
@scheduled_events = []
|
|
83
80
|
# The reason the trigger gives the process for what it dispatches,
|
|
@@ -90,9 +87,7 @@ module Bluepill
|
|
|
90
87
|
# @return [void]
|
|
91
88
|
#
|
|
92
89
|
# @api private
|
|
93
|
-
def reset!
|
|
94
|
-
cancel_all_events
|
|
95
|
-
end
|
|
90
|
+
def reset! = cancel_all_events
|
|
96
91
|
|
|
97
92
|
# React to a state transition, before it happens
|
|
98
93
|
#
|
|
@@ -103,9 +98,7 @@ module Bluepill
|
|
|
103
98
|
# @raise [RuntimeError] unless a subclass implements it
|
|
104
99
|
#
|
|
105
100
|
# @api private
|
|
106
|
-
def notify(_transition)
|
|
107
|
-
raise "Implement in subclass"
|
|
108
|
-
end
|
|
101
|
+
def notify(_transition) = raise("Implement in subclass")
|
|
109
102
|
|
|
110
103
|
# Send an event to the process, with the trigger's name as the reason
|
|
111
104
|
#
|
|
@@ -114,9 +107,7 @@ module Bluepill
|
|
|
114
107
|
# @return [Boolean] whether the process made the transition
|
|
115
108
|
#
|
|
116
109
|
# @api private
|
|
117
|
-
def dispatch!(event)
|
|
118
|
-
process.dispatch!(event, @reason)
|
|
119
|
-
end
|
|
110
|
+
def dispatch!(event) = process.dispatch!(event, @reason)
|
|
120
111
|
|
|
121
112
|
# Dispatch an event after a delay, in a thread of its own
|
|
122
113
|
#
|
|
@@ -153,8 +144,7 @@ module Bluepill
|
|
|
153
144
|
scheduled_events.delete_if { |_, t| t.equal?(Thread.current) }
|
|
154
145
|
end
|
|
155
146
|
rescue => e
|
|
156
|
-
|
|
157
|
-
logger.err(e.backtrace.to_a.join("\n"))
|
|
147
|
+
Util.log_error(logger, "#{@reason} failed to dispatch #{event}", e)
|
|
158
148
|
end
|
|
159
149
|
|
|
160
150
|
# Cancel every scheduled event, killing its thread
|
|
@@ -55,18 +55,22 @@ module Bluepill
|
|
|
55
55
|
# Create the trigger for a process
|
|
56
56
|
#
|
|
57
57
|
# @param [Process] process the process to watch
|
|
58
|
-
# @param [
|
|
59
|
-
#
|
|
58
|
+
# @param [Integer] times how many starts and restarts are flapping
|
|
59
|
+
# @param [Numeric] within the seconds they must happen within
|
|
60
|
+
# @param [Numeric] retry_in the seconds to wait before starting the
|
|
61
|
+
# process again, or 0 to leave it unmonitored
|
|
62
|
+
# @param [Hash{Symbol => Object}] options the other options, with
|
|
63
|
+
# :logger
|
|
60
64
|
#
|
|
61
65
|
# @return [void]
|
|
62
66
|
#
|
|
63
67
|
# @api private
|
|
64
|
-
def initialize(process,
|
|
65
|
-
@times =
|
|
66
|
-
@within =
|
|
67
|
-
@retry_in =
|
|
68
|
-
@timeline = Util::RotationalArray.new(
|
|
69
|
-
super
|
|
68
|
+
def initialize(process, times: 5, within: 1, retry_in: 5, **options)
|
|
69
|
+
@times = times
|
|
70
|
+
@within = within
|
|
71
|
+
@retry_in = retry_in
|
|
72
|
+
@timeline = Util::RotationalArray.new(times)
|
|
73
|
+
super(process, **options)
|
|
70
74
|
end
|
|
71
75
|
|
|
72
76
|
# Record a start or a restart, and check whether the process flaps
|
data/lib/bluepill/units.rb
CHANGED
|
@@ -36,9 +36,7 @@ module Bluepill
|
|
|
36
36
|
# @return [Numeric] the number itself
|
|
37
37
|
#
|
|
38
38
|
# @api public
|
|
39
|
-
def seconds
|
|
40
|
-
self
|
|
41
|
-
end
|
|
39
|
+
def seconds = self
|
|
42
40
|
alias_method :second, :seconds
|
|
43
41
|
|
|
44
42
|
# Return this many minutes, in seconds
|
|
@@ -49,9 +47,7 @@ module Bluepill
|
|
|
49
47
|
# @return [Numeric] the number of seconds
|
|
50
48
|
#
|
|
51
49
|
# @api public
|
|
52
|
-
def minutes
|
|
53
|
-
self * MINUTE
|
|
54
|
-
end
|
|
50
|
+
def minutes = self * MINUTE
|
|
55
51
|
alias_method :minute, :minutes
|
|
56
52
|
|
|
57
53
|
# Return this many hours, in seconds
|
|
@@ -62,9 +58,7 @@ module Bluepill
|
|
|
62
58
|
# @return [Numeric] the number of seconds
|
|
63
59
|
#
|
|
64
60
|
# @api public
|
|
65
|
-
def hours
|
|
66
|
-
self * HOUR
|
|
67
|
-
end
|
|
61
|
+
def hours = self * HOUR
|
|
68
62
|
alias_method :hour, :hours
|
|
69
63
|
|
|
70
64
|
# Return this many days, in seconds
|
|
@@ -75,9 +69,7 @@ module Bluepill
|
|
|
75
69
|
# @return [Numeric] the number of seconds
|
|
76
70
|
#
|
|
77
71
|
# @api public
|
|
78
|
-
def days
|
|
79
|
-
self * DAY
|
|
80
|
-
end
|
|
72
|
+
def days = self * DAY
|
|
81
73
|
alias_method :day, :days
|
|
82
74
|
|
|
83
75
|
# Return this many kilobytes, in bytes
|
|
@@ -88,9 +80,7 @@ module Bluepill
|
|
|
88
80
|
# @return [Numeric] the number of bytes
|
|
89
81
|
#
|
|
90
82
|
# @api public
|
|
91
|
-
def kilobytes
|
|
92
|
-
self * KILOBYTE
|
|
93
|
-
end
|
|
83
|
+
def kilobytes = self * KILOBYTE
|
|
94
84
|
alias_method :kilobyte, :kilobytes
|
|
95
85
|
|
|
96
86
|
# Return this many megabytes, in bytes
|
|
@@ -101,9 +91,7 @@ module Bluepill
|
|
|
101
91
|
# @return [Numeric] the number of bytes
|
|
102
92
|
#
|
|
103
93
|
# @api public
|
|
104
|
-
def megabytes
|
|
105
|
-
self * MEGABYTE
|
|
106
|
-
end
|
|
94
|
+
def megabytes = self * MEGABYTE
|
|
107
95
|
alias_method :megabyte, :megabytes
|
|
108
96
|
|
|
109
97
|
# Return this many gigabytes, in bytes
|
|
@@ -114,9 +102,7 @@ module Bluepill
|
|
|
114
102
|
# @return [Numeric] the number of bytes
|
|
115
103
|
#
|
|
116
104
|
# @api public
|
|
117
|
-
def gigabytes
|
|
118
|
-
self * GIGABYTE
|
|
119
|
-
end
|
|
105
|
+
def gigabytes = self * GIGABYTE
|
|
120
106
|
alias_method :gigabyte, :gigabytes
|
|
121
107
|
end
|
|
122
108
|
end
|
|
@@ -0,0 +1,28 @@
|
|
|
1
|
+
# frozen_string_literal: true
|
|
2
|
+
|
|
3
|
+
module Bluepill
|
|
4
|
+
module Util
|
|
5
|
+
# Log an error at err, with a line of its own for each backtrace line
|
|
6
|
+
#
|
|
7
|
+
# The daemon rescues what a check, a trigger or a tick raises, so the
|
|
8
|
+
# log is the only place the error shows.
|
|
9
|
+
#
|
|
10
|
+
# @example
|
|
11
|
+
# Bluepill::Util.log_error(logger, "mem_usage check failed", error)
|
|
12
|
+
# # mem_usage check failed: RuntimeError: boom
|
|
13
|
+
# # /path/to/check.rb:12:in `run'
|
|
14
|
+
#
|
|
15
|
+
# @param [Logger] logger the logger to write to
|
|
16
|
+
# @param [String] summary what failed, which the error's class and
|
|
17
|
+
# message follow
|
|
18
|
+
# @param [Exception] error the error
|
|
19
|
+
#
|
|
20
|
+
# @return [void]
|
|
21
|
+
#
|
|
22
|
+
# @api private
|
|
23
|
+
def self.log_error(logger, summary, error)
|
|
24
|
+
logger.err("#{summary}: #{error.class}: #{error}")
|
|
25
|
+
error.backtrace&.each { |line| logger.err(line) }
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|