resurrected_god 0.14.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/Announce.txt +135 -0
- data/Gemfile +5 -0
- data/LICENSE +22 -0
- data/README.md +33 -0
- data/Rakefile +129 -0
- data/bin/god +134 -0
- data/doc/god.asciidoc +1592 -0
- data/doc/intro.asciidoc +20 -0
- data/ext/god/.gitignore +5 -0
- data/ext/god/extconf.rb +56 -0
- data/ext/god/kqueue_handler.c +133 -0
- data/ext/god/netlink_handler.c +182 -0
- data/lib/god/behavior.rb +52 -0
- data/lib/god/behaviors/clean_pid_file.rb +21 -0
- data/lib/god/behaviors/clean_unix_socket.rb +21 -0
- data/lib/god/behaviors/notify_when_flapping.rb +51 -0
- data/lib/god/cli/command.rb +268 -0
- data/lib/god/cli/run.rb +170 -0
- data/lib/god/cli/version.rb +23 -0
- data/lib/god/compat19.rb +33 -0
- data/lib/god/condition.rb +96 -0
- data/lib/god/conditions/always.rb +36 -0
- data/lib/god/conditions/complex.rb +86 -0
- data/lib/god/conditions/cpu_usage.rb +80 -0
- data/lib/god/conditions/degrading_lambda.rb +52 -0
- data/lib/god/conditions/disk_usage.rb +32 -0
- data/lib/god/conditions/file_mtime.rb +28 -0
- data/lib/god/conditions/file_touched.rb +44 -0
- data/lib/god/conditions/flapping.rb +128 -0
- data/lib/god/conditions/http_response_code.rb +184 -0
- data/lib/god/conditions/lambda.rb +25 -0
- data/lib/god/conditions/memory_usage.rb +82 -0
- data/lib/god/conditions/process_exits.rb +66 -0
- data/lib/god/conditions/process_running.rb +63 -0
- data/lib/god/conditions/socket_responding.rb +142 -0
- data/lib/god/conditions/tries.rb +44 -0
- data/lib/god/configurable.rb +57 -0
- data/lib/god/contact.rb +114 -0
- data/lib/god/contacts/airbrake.rb +44 -0
- data/lib/god/contacts/campfire.rb +121 -0
- data/lib/god/contacts/email.rb +130 -0
- data/lib/god/contacts/hipchat.rb +117 -0
- data/lib/god/contacts/jabber.rb +75 -0
- data/lib/god/contacts/prowl.rb +57 -0
- data/lib/god/contacts/scout.rb +55 -0
- data/lib/god/contacts/sensu.rb +59 -0
- data/lib/god/contacts/slack.rb +98 -0
- data/lib/god/contacts/statsd.rb +46 -0
- data/lib/god/contacts/twitter.rb +51 -0
- data/lib/god/contacts/webhook.rb +74 -0
- data/lib/god/driver.rb +238 -0
- data/lib/god/errors.rb +24 -0
- data/lib/god/event_handler.rb +112 -0
- data/lib/god/event_handlers/dummy_handler.rb +13 -0
- data/lib/god/event_handlers/kqueue_handler.rb +17 -0
- data/lib/god/event_handlers/netlink_handler.rb +13 -0
- data/lib/god/logger.rb +109 -0
- data/lib/god/metric.rb +87 -0
- data/lib/god/process.rb +381 -0
- data/lib/god/registry.rb +32 -0
- data/lib/god/simple_logger.rb +59 -0
- data/lib/god/socket.rb +113 -0
- data/lib/god/sugar.rb +62 -0
- data/lib/god/sys_logger.rb +45 -0
- data/lib/god/system/portable_poller.rb +42 -0
- data/lib/god/system/process.rb +50 -0
- data/lib/god/system/slash_proc_poller.rb +92 -0
- data/lib/god/task.rb +552 -0
- data/lib/god/timeline.rb +25 -0
- data/lib/god/trigger.rb +43 -0
- data/lib/god/version.rb +4 -0
- data/lib/god/watch.rb +340 -0
- data/lib/god.rb +777 -0
- data/test/configs/child_events/child_events.god +44 -0
- data/test/configs/child_events/simple_server.rb +3 -0
- data/test/configs/child_polls/child_polls.god +37 -0
- data/test/configs/child_polls/simple_server.rb +12 -0
- data/test/configs/complex/complex.god +59 -0
- data/test/configs/complex/simple_server.rb +3 -0
- data/test/configs/contact/contact.god +118 -0
- data/test/configs/contact/simple_server.rb +3 -0
- data/test/configs/daemon_events/daemon_events.god +37 -0
- data/test/configs/daemon_events/simple_server.rb +8 -0
- data/test/configs/daemon_events/simple_server_stop.rb +11 -0
- data/test/configs/daemon_polls/daemon_polls.god +17 -0
- data/test/configs/daemon_polls/simple_server.rb +6 -0
- data/test/configs/degrading_lambda/degrading_lambda.god +31 -0
- data/test/configs/degrading_lambda/tcp_server.rb +15 -0
- data/test/configs/keepalive/keepalive.god +9 -0
- data/test/configs/keepalive/keepalive.rb +12 -0
- data/test/configs/lifecycle/lifecycle.god +25 -0
- data/test/configs/matias/matias.god +50 -0
- data/test/configs/real.rb +59 -0
- data/test/configs/running_load/running_load.god +16 -0
- data/test/configs/stop_options/simple_server.rb +12 -0
- data/test/configs/stop_options/stop_options.god +39 -0
- data/test/configs/stress/simple_server.rb +3 -0
- data/test/configs/stress/stress.god +15 -0
- data/test/configs/task/logs/.placeholder +0 -0
- data/test/configs/task/task.god +26 -0
- data/test/configs/test.rb +61 -0
- data/test/configs/usr1_trapper.rb +10 -0
- data/test/helper.rb +172 -0
- data/test/suite.rb +6 -0
- data/test/test_airbrake.rb +14 -0
- data/test/test_behavior.rb +18 -0
- data/test/test_campfire.rb +22 -0
- data/test/test_condition.rb +52 -0
- data/test/test_conditions_disk_usage.rb +50 -0
- data/test/test_conditions_http_response_code.rb +109 -0
- data/test/test_conditions_process_running.rb +40 -0
- data/test/test_conditions_socket_responding.rb +176 -0
- data/test/test_conditions_tries.rb +67 -0
- data/test/test_contact.rb +109 -0
- data/test/test_driver.rb +26 -0
- data/test/test_email.rb +34 -0
- data/test/test_event_handler.rb +82 -0
- data/test/test_god.rb +710 -0
- data/test/test_god_system.rb +201 -0
- data/test/test_handlers_kqueue_handler.rb +16 -0
- data/test/test_hipchat.rb +23 -0
- data/test/test_jabber.rb +29 -0
- data/test/test_logger.rb +55 -0
- data/test/test_metric.rb +74 -0
- data/test/test_process.rb +263 -0
- data/test/test_prowl.rb +15 -0
- data/test/test_registry.rb +15 -0
- data/test/test_sensu.rb +11 -0
- data/test/test_slack.rb +57 -0
- data/test/test_socket.rb +34 -0
- data/test/test_statsd.rb +22 -0
- data/test/test_sugar.rb +42 -0
- data/test/test_system_portable_poller.rb +17 -0
- data/test/test_system_process.rb +30 -0
- data/test/test_task.rb +246 -0
- data/test/test_timeline.rb +37 -0
- data/test/test_trigger.rb +63 -0
- data/test/test_watch.rb +286 -0
- data/test/test_webhook.rb +22 -0
- metadata +476 -0
data/lib/god/watch.rb
ADDED
@@ -0,0 +1,340 @@
|
|
1
|
+
require 'etc'
|
2
|
+
require 'forwardable'
|
3
|
+
|
4
|
+
module God
|
5
|
+
# The Watch class is a specialized Task that handles standard process
|
6
|
+
# workflows. It has four states: init, up, start, and restart.
|
7
|
+
class Watch < Task
|
8
|
+
# The Array of Symbol valid task states.
|
9
|
+
VALID_STATES = [:init, :up, :start, :restart]
|
10
|
+
|
11
|
+
# The Sybmol initial state.
|
12
|
+
INITIAL_STATE = :init
|
13
|
+
|
14
|
+
# Public: The grace period for this process (seconds).
|
15
|
+
attr_accessor :grace
|
16
|
+
|
17
|
+
# Public: The start grace period (seconds).
|
18
|
+
attr_accessor :start_grace
|
19
|
+
|
20
|
+
# Public: The stop grace period (seconds).
|
21
|
+
attr_accessor :stop_grace
|
22
|
+
|
23
|
+
# Public: The restart grace period (seconds).
|
24
|
+
attr_accessor :restart_grace
|
25
|
+
|
26
|
+
# Public: God::Process delegators. See lib/god/process.rb for docs.
|
27
|
+
extend Forwardable
|
28
|
+
def_delegators :@process, :name, :uid, :gid, :start, :stop, :restart, :dir,
|
29
|
+
:name=, :uid=, :gid=, :start=, :stop=, :restart=,
|
30
|
+
:dir=, :pid_file, :pid_file=, :log, :log=,
|
31
|
+
:log_cmd, :log_cmd=, :err_log, :err_log=,
|
32
|
+
:err_log_cmd, :err_log_cmd=, :alive?, :pid,
|
33
|
+
:unix_socket, :unix_socket=, :chroot, :chroot=,
|
34
|
+
:env, :env=, :signal, :stop_timeout=,
|
35
|
+
:stop_signal=, :umask, :umask=
|
36
|
+
|
37
|
+
# Initialize a new Watch instance.
|
38
|
+
def initialize
|
39
|
+
super
|
40
|
+
|
41
|
+
# This God::Process instance holds information specific to the process.
|
42
|
+
@process = God::Process.new
|
43
|
+
|
44
|
+
# Valid states.
|
45
|
+
self.valid_states = VALID_STATES
|
46
|
+
self.initial_state = INITIAL_STATE
|
47
|
+
|
48
|
+
# No grace period by default.
|
49
|
+
self.grace = self.start_grace = self.stop_grace = self.restart_grace = 0
|
50
|
+
end
|
51
|
+
|
52
|
+
# Is this Watch valid?
|
53
|
+
#
|
54
|
+
# Returns true if the Watch is valid, false if not.
|
55
|
+
def valid?
|
56
|
+
super && @process.valid?
|
57
|
+
end
|
58
|
+
|
59
|
+
###########################################################################
|
60
|
+
#
|
61
|
+
# Behavior
|
62
|
+
#
|
63
|
+
###########################################################################
|
64
|
+
|
65
|
+
# Public: Add a behavior to this Watch. See lib/god/behavior.rb.
|
66
|
+
#
|
67
|
+
# kind - The Symbol name of the Behavior to add.
|
68
|
+
#
|
69
|
+
# Yields the newly instantiated Behavior.
|
70
|
+
#
|
71
|
+
# Returns nothing.
|
72
|
+
def behavior(kind)
|
73
|
+
# Create the behavior.
|
74
|
+
begin
|
75
|
+
b = Behavior.generate(kind, self)
|
76
|
+
rescue NoSuchBehaviorError => e
|
77
|
+
abort e.message
|
78
|
+
end
|
79
|
+
|
80
|
+
# Send to block so config can set attributes.
|
81
|
+
yield(b) if block_given?
|
82
|
+
|
83
|
+
# Abort if the Behavior is invalid, the Behavior will have printed
|
84
|
+
# out its own error messages by now.
|
85
|
+
abort unless b.valid?
|
86
|
+
|
87
|
+
self.behaviors << b
|
88
|
+
end
|
89
|
+
|
90
|
+
###########################################################################
|
91
|
+
#
|
92
|
+
# Quickstart mode
|
93
|
+
#
|
94
|
+
###########################################################################
|
95
|
+
|
96
|
+
# Default Integer interval at which keepalive will runn poll checks.
|
97
|
+
DEFAULT_KEEPALIVE_INTERVAL = 5.seconds
|
98
|
+
|
99
|
+
# Default Integer or Array of Integers specification of how many times the
|
100
|
+
# memory condition must fail before triggering.
|
101
|
+
DEFAULT_KEEPALIVE_MEMORY_TIMES = [3, 5]
|
102
|
+
|
103
|
+
# Default Integer or Array of Integers specification of how many times the
|
104
|
+
# CPU condition must fail before triggering.
|
105
|
+
DEFAULT_KEEPALIVE_CPU_TIMES = [3, 5]
|
106
|
+
|
107
|
+
# Public: A set of conditions for easily getting started with simple watch
|
108
|
+
# scenarios. Keepalive is intended for use by beginners or on processes
|
109
|
+
# that do not need very sophisticated monitoring.
|
110
|
+
#
|
111
|
+
# If events are enabled, it will use the :process_exit event to determine
|
112
|
+
# if a process fails. Otherwise it will use the :process_running poll.
|
113
|
+
#
|
114
|
+
# options - The option Hash. Possible values are:
|
115
|
+
# :interval - The Integer number of seconds on which to poll
|
116
|
+
# for process status. Affects CPU, memory, and
|
117
|
+
# :process_running conditions (if used).
|
118
|
+
# Default: 5.seconds.
|
119
|
+
# :memory_max - The Integer memory max. A bare integer means
|
120
|
+
# kilobytes. You may use Numeric.kilobytes,
|
121
|
+
# Numeric#megabytes, and Numeric#gigabytes to
|
122
|
+
# makes things more clear.
|
123
|
+
# :memory_times - If :memory_max is set, :memory_times can be
|
124
|
+
# set to either an Integer or a 2 element
|
125
|
+
# Integer Array to specify the number of times
|
126
|
+
# the memory condition must fail. Examples:
|
127
|
+
# 3 (three times), [3, 5] (three out of any five
|
128
|
+
# checks). Default: [3, 5].
|
129
|
+
# :cpu_max - The Integer CPU percentage max. Range is
|
130
|
+
# 0 to 100. You may use the Numberic#percent
|
131
|
+
# sugar to clarify e.g. 50.percent.
|
132
|
+
# :cpu_times - If :cpu_max is set, :cpu_times can be
|
133
|
+
# set to either an Integer or a 2 element
|
134
|
+
# Integer Array to specify the number of times
|
135
|
+
# the memory condition must fail. Examples:
|
136
|
+
# 3 (three times), [3, 5] (three out of any five
|
137
|
+
# checks). Default: [3, 5].
|
138
|
+
def keepalive(options = {})
|
139
|
+
if God::EventHandler.loaded?
|
140
|
+
self.transition(:init, { true => :up, false => :start }) do |on|
|
141
|
+
on.condition(:process_running) do |c|
|
142
|
+
c.interval = options[:interval] || DEFAULT_KEEPALIVE_INTERVAL
|
143
|
+
c.running = true
|
144
|
+
end
|
145
|
+
end
|
146
|
+
|
147
|
+
self.transition([:start, :restart], :up) do |on|
|
148
|
+
on.condition(:process_running) do |c|
|
149
|
+
c.interval = options[:interval] || DEFAULT_KEEPALIVE_INTERVAL
|
150
|
+
c.running = true
|
151
|
+
end
|
152
|
+
end
|
153
|
+
|
154
|
+
self.transition(:up, :start) do |on|
|
155
|
+
on.condition(:process_exits)
|
156
|
+
end
|
157
|
+
else
|
158
|
+
self.start_if do |start|
|
159
|
+
start.condition(:process_running) do |c|
|
160
|
+
c.interval = options[:interval] || DEFAULT_KEEPALIVE_INTERVAL
|
161
|
+
c.running = false
|
162
|
+
end
|
163
|
+
end
|
164
|
+
end
|
165
|
+
|
166
|
+
self.restart_if do |restart|
|
167
|
+
if options[:memory_max]
|
168
|
+
restart.condition(:memory_usage) do |c|
|
169
|
+
c.interval = options[:interval] || DEFAULT_KEEPALIVE_INTERVAL
|
170
|
+
c.above = options[:memory_max]
|
171
|
+
c.times = options[:memory_times] || DEFAULT_KEEPALIVE_MEMORY_TIMES
|
172
|
+
end
|
173
|
+
end
|
174
|
+
|
175
|
+
if options[:cpu_max]
|
176
|
+
restart.condition(:cpu_usage) do |c|
|
177
|
+
c.interval = options[:interval] || DEFAULT_KEEPALIVE_INTERVAL
|
178
|
+
c.above = options[:cpu_max]
|
179
|
+
c.times = options[:cpu_times] || DEFAULT_KEEPALIVE_CPU_TIMES
|
180
|
+
end
|
181
|
+
end
|
182
|
+
end
|
183
|
+
end
|
184
|
+
|
185
|
+
###########################################################################
|
186
|
+
#
|
187
|
+
# Simple mode
|
188
|
+
#
|
189
|
+
###########################################################################
|
190
|
+
|
191
|
+
# Public: Start the process if any of the given conditions are triggered.
|
192
|
+
#
|
193
|
+
# Yields the Metric upon which conditions can be added.
|
194
|
+
#
|
195
|
+
# Returns nothing.
|
196
|
+
def start_if
|
197
|
+
self.transition(:up, :start) do |on|
|
198
|
+
yield(on)
|
199
|
+
end
|
200
|
+
end
|
201
|
+
|
202
|
+
# Public: Restart the process if any of the given conditions are triggered.
|
203
|
+
#
|
204
|
+
# Yields the Metric upon which conditions can be added.
|
205
|
+
#
|
206
|
+
# Returns nothing.
|
207
|
+
def restart_if
|
208
|
+
self.transition(:up, :restart) do |on|
|
209
|
+
yield(on)
|
210
|
+
end
|
211
|
+
end
|
212
|
+
|
213
|
+
# Public: Stop the process if any of the given conditions are triggered.
|
214
|
+
#
|
215
|
+
# Yields the Metric upon which conditions can be added.
|
216
|
+
#
|
217
|
+
# Returns nothing.
|
218
|
+
def stop_if
|
219
|
+
self.transition(:up, :stop) do |on|
|
220
|
+
yield(on)
|
221
|
+
end
|
222
|
+
end
|
223
|
+
|
224
|
+
###########################################################################
|
225
|
+
#
|
226
|
+
# Lifecycle
|
227
|
+
#
|
228
|
+
###########################################################################
|
229
|
+
|
230
|
+
# Enable monitoring. Start at the first available of the init or up states.
|
231
|
+
#
|
232
|
+
# Returns nothing.
|
233
|
+
def monitor
|
234
|
+
if !self.metrics[:init].empty?
|
235
|
+
self.move(:init)
|
236
|
+
else
|
237
|
+
self.move(:up)
|
238
|
+
end
|
239
|
+
end
|
240
|
+
|
241
|
+
###########################################################################
|
242
|
+
#
|
243
|
+
# Actions
|
244
|
+
#
|
245
|
+
###########################################################################
|
246
|
+
|
247
|
+
# Perform an action.
|
248
|
+
#
|
249
|
+
# a - The Symbol action to perform. One of :start, :restart, :stop.
|
250
|
+
# c - The Condition.
|
251
|
+
#
|
252
|
+
# Returns this Watch.
|
253
|
+
def action(a, c = nil)
|
254
|
+
if !self.driver.in_driver_context?
|
255
|
+
# Called from outside Driver. Send an async message to Driver.
|
256
|
+
self.driver.message(:action, [a, c])
|
257
|
+
else
|
258
|
+
# Called from within Driver.
|
259
|
+
case a
|
260
|
+
when :start
|
261
|
+
call_action(c, :start)
|
262
|
+
sleep(self.start_grace + self.grace)
|
263
|
+
when :restart
|
264
|
+
if self.restart
|
265
|
+
call_action(c, :restart)
|
266
|
+
else
|
267
|
+
action(:stop, c)
|
268
|
+
action(:start, c)
|
269
|
+
end
|
270
|
+
sleep(self.restart_grace + self.grace)
|
271
|
+
when :stop
|
272
|
+
call_action(c, :stop)
|
273
|
+
sleep(self.stop_grace + self.grace)
|
274
|
+
end
|
275
|
+
end
|
276
|
+
|
277
|
+
self
|
278
|
+
end
|
279
|
+
|
280
|
+
# Perform the specifics of the action.
|
281
|
+
#
|
282
|
+
# condition - The Condition.
|
283
|
+
# action - The Symbol action.
|
284
|
+
#
|
285
|
+
# Returns nothing.
|
286
|
+
def call_action(condition, action)
|
287
|
+
# Before.
|
288
|
+
before_items = self.behaviors
|
289
|
+
before_items += [condition] if condition
|
290
|
+
before_items.each do |b|
|
291
|
+
info = b.send("before_#{action}")
|
292
|
+
if info
|
293
|
+
msg = "#{self.name} before_#{action}: #{info} (#{b.base_name})"
|
294
|
+
applog(self, :info, msg)
|
295
|
+
end
|
296
|
+
end
|
297
|
+
|
298
|
+
# Log.
|
299
|
+
if self.send(action)
|
300
|
+
msg = "#{self.name} #{action}: #{self.send(action).to_s}"
|
301
|
+
applog(self, :info, msg)
|
302
|
+
end
|
303
|
+
|
304
|
+
# Execute.
|
305
|
+
@process.call_action(action)
|
306
|
+
|
307
|
+
# After.
|
308
|
+
after_items = self.behaviors
|
309
|
+
after_items += [condition] if condition
|
310
|
+
after_items.each do |b|
|
311
|
+
info = b.send("after_#{action}")
|
312
|
+
if info
|
313
|
+
msg = "#{self.name} after_#{action}: #{info} (#{b.base_name})"
|
314
|
+
applog(self, :info, msg)
|
315
|
+
end
|
316
|
+
end
|
317
|
+
end
|
318
|
+
|
319
|
+
###########################################################################
|
320
|
+
#
|
321
|
+
# Registration
|
322
|
+
#
|
323
|
+
###########################################################################
|
324
|
+
|
325
|
+
# Register the Process in the global process registry.
|
326
|
+
#
|
327
|
+
# Returns nothing.
|
328
|
+
def register!
|
329
|
+
God.registry.add(@process)
|
330
|
+
end
|
331
|
+
|
332
|
+
# Unregister the Process in the global process registry.
|
333
|
+
#
|
334
|
+
# Returns nothing.
|
335
|
+
def unregister!
|
336
|
+
God.registry.remove(@process)
|
337
|
+
super
|
338
|
+
end
|
339
|
+
end
|
340
|
+
end
|