mojombo-god 0.7.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +255 -0
- data/Manifest.txt +107 -0
- data/README.txt +59 -0
- data/Rakefile +35 -0
- data/bin/god +127 -0
- data/examples/events.god +84 -0
- data/examples/gravatar.god +54 -0
- data/examples/single.god +66 -0
- data/ext/god/extconf.rb +55 -0
- data/ext/god/kqueue_handler.c +123 -0
- data/ext/god/netlink_handler.c +167 -0
- data/init/god +42 -0
- data/lib/god.rb +644 -0
- data/lib/god/behavior.rb +52 -0
- data/lib/god/behaviors/clean_pid_file.rb +21 -0
- data/lib/god/behaviors/clean_unix_socket.rb +21 -0
- data/lib/god/behaviors/notify_when_flapping.rb +51 -0
- data/lib/god/cli/command.rb +206 -0
- data/lib/god/cli/run.rb +177 -0
- data/lib/god/cli/version.rb +23 -0
- data/lib/god/condition.rb +96 -0
- data/lib/god/conditions/always.rb +23 -0
- data/lib/god/conditions/complex.rb +86 -0
- data/lib/god/conditions/cpu_usage.rb +80 -0
- data/lib/god/conditions/degrading_lambda.rb +52 -0
- data/lib/god/conditions/disk_usage.rb +27 -0
- data/lib/god/conditions/flapping.rb +128 -0
- data/lib/god/conditions/http_response_code.rb +168 -0
- data/lib/god/conditions/lambda.rb +25 -0
- data/lib/god/conditions/memory_usage.rb +82 -0
- data/lib/god/conditions/process_exits.rb +72 -0
- data/lib/god/conditions/process_running.rb +74 -0
- data/lib/god/conditions/tries.rb +44 -0
- data/lib/god/configurable.rb +57 -0
- data/lib/god/contact.rb +106 -0
- data/lib/god/contacts/email.rb +95 -0
- data/lib/god/dependency_graph.rb +41 -0
- data/lib/god/diagnostics.rb +37 -0
- data/lib/god/driver.rb +108 -0
- data/lib/god/errors.rb +24 -0
- data/lib/god/event_handler.rb +111 -0
- data/lib/god/event_handlers/dummy_handler.rb +13 -0
- data/lib/god/event_handlers/kqueue_handler.rb +17 -0
- data/lib/god/event_handlers/netlink_handler.rb +13 -0
- data/lib/god/logger.rb +120 -0
- data/lib/god/metric.rb +59 -0
- data/lib/god/process.rb +325 -0
- data/lib/god/registry.rb +32 -0
- data/lib/god/simple_logger.rb +53 -0
- data/lib/god/socket.rb +96 -0
- data/lib/god/sugar.rb +47 -0
- data/lib/god/system/portable_poller.rb +42 -0
- data/lib/god/system/process.rb +42 -0
- data/lib/god/system/slash_proc_poller.rb +82 -0
- data/lib/god/task.rb +487 -0
- data/lib/god/timeline.rb +25 -0
- data/lib/god/trigger.rb +43 -0
- data/lib/god/watch.rb +183 -0
- data/test/configs/child_events/child_events.god +44 -0
- data/test/configs/child_events/simple_server.rb +3 -0
- data/test/configs/child_polls/child_polls.god +37 -0
- data/test/configs/child_polls/simple_server.rb +12 -0
- data/test/configs/complex/complex.god +59 -0
- data/test/configs/complex/simple_server.rb +3 -0
- data/test/configs/contact/contact.god +74 -0
- data/test/configs/contact/simple_server.rb +3 -0
- data/test/configs/daemon_events/daemon_events.god +37 -0
- data/test/configs/daemon_events/simple_server.rb +8 -0
- data/test/configs/daemon_events/simple_server_stop.rb +11 -0
- data/test/configs/daemon_polls/daemon_polls.god +17 -0
- data/test/configs/daemon_polls/simple_server.rb +6 -0
- data/test/configs/degrading_lambda/degrading_lambda.god +31 -0
- data/test/configs/degrading_lambda/tcp_server.rb +15 -0
- data/test/configs/matias/matias.god +50 -0
- data/test/configs/real.rb +59 -0
- data/test/configs/running_load/running_load.god +16 -0
- data/test/configs/stress/simple_server.rb +3 -0
- data/test/configs/stress/stress.god +15 -0
- data/test/configs/task/logs/.placeholder +0 -0
- data/test/configs/task/task.god +26 -0
- data/test/configs/test.rb +61 -0
- data/test/helper.rb +151 -0
- data/test/suite.rb +6 -0
- data/test/test_behavior.rb +21 -0
- data/test/test_condition.rb +50 -0
- data/test/test_conditions_disk_usage.rb +56 -0
- data/test/test_conditions_http_response_code.rb +109 -0
- data/test/test_conditions_process_running.rb +44 -0
- data/test/test_conditions_tries.rb +67 -0
- data/test/test_contact.rb +109 -0
- data/test/test_dependency_graph.rb +62 -0
- data/test/test_driver.rb +11 -0
- data/test/test_event_handler.rb +80 -0
- data/test/test_god.rb +598 -0
- data/test/test_handlers_kqueue_handler.rb +16 -0
- data/test/test_logger.rb +63 -0
- data/test/test_metric.rb +72 -0
- data/test/test_process.rb +246 -0
- data/test/test_registry.rb +15 -0
- data/test/test_socket.rb +42 -0
- data/test/test_sugar.rb +42 -0
- data/test/test_system_portable_poller.rb +17 -0
- data/test/test_system_process.rb +30 -0
- data/test/test_task.rb +262 -0
- data/test/test_timeline.rb +37 -0
- data/test/test_trigger.rb +59 -0
- data/test/test_watch.rb +279 -0
- metadata +186 -0
@@ -0,0 +1,23 @@
|
|
1
|
+
module God
|
2
|
+
module CLI
|
3
|
+
|
4
|
+
class Version
|
5
|
+
def self.version
|
6
|
+
require 'god'
|
7
|
+
|
8
|
+
# print version
|
9
|
+
puts "Version #{God::VERSION}"
|
10
|
+
exit
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.version_extended
|
14
|
+
puts "Version: #{God::VERSION}"
|
15
|
+
puts "Polls: enabled"
|
16
|
+
puts "Events: " + God::EventHandler.event_system
|
17
|
+
|
18
|
+
exit
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
module God
|
2
|
+
|
3
|
+
class Condition < Behavior
|
4
|
+
attr_accessor :transition, :notify, :info, :phase
|
5
|
+
|
6
|
+
# Generate a Condition of the given kind. The proper class if found by camel casing the
|
7
|
+
# kind (which is given as an underscored symbol).
|
8
|
+
# +kind+ is the underscored symbol representing the class (e.g. :foo_bar for God::Conditions::FooBar)
|
9
|
+
def self.generate(kind, watch)
|
10
|
+
sym = kind.to_s.capitalize.gsub(/_(.)/){$1.upcase}.intern
|
11
|
+
c = God::Conditions.const_get(sym).new
|
12
|
+
|
13
|
+
unless c.kind_of?(PollCondition) || c.kind_of?(EventCondition) || c.kind_of?(TriggerCondition)
|
14
|
+
abort "Condition '#{c.class.name}' must subclass God::PollCondition, God::EventCondition, or God::TriggerCondition"
|
15
|
+
end
|
16
|
+
|
17
|
+
if !EventHandler.loaded? && c.kind_of?(EventCondition)
|
18
|
+
abort "Condition '#{c.class.name}' requires an event system but none has been loaded"
|
19
|
+
end
|
20
|
+
|
21
|
+
c.watch = watch
|
22
|
+
c
|
23
|
+
rescue NameError
|
24
|
+
raise NoSuchConditionError.new("No Condition found with the class name God::Conditions::#{sym}")
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.valid?(condition)
|
28
|
+
valid = true
|
29
|
+
if condition.notify
|
30
|
+
begin
|
31
|
+
Contact.normalize(condition.notify)
|
32
|
+
rescue ArgumentError => e
|
33
|
+
valid &= Configurable.complain("Attribute 'notify' " + e.message, condition)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
valid
|
37
|
+
end
|
38
|
+
|
39
|
+
# Construct the friendly name of this Condition, looks like:
|
40
|
+
#
|
41
|
+
# Condition FooBar on Watch 'baz'
|
42
|
+
def friendly_name
|
43
|
+
"Condition #{self.class.name.split('::').last} on Watch '#{self.watch.name}'"
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
class PollCondition < Condition
|
48
|
+
# all poll conditions can specify a poll interval
|
49
|
+
attr_accessor :interval
|
50
|
+
|
51
|
+
# Override this method in your Conditions (optional)
|
52
|
+
def before
|
53
|
+
end
|
54
|
+
|
55
|
+
# Override this method in your Conditions (mandatory)
|
56
|
+
#
|
57
|
+
# Return true if the test passes (everything is ok)
|
58
|
+
# Return false otherwise
|
59
|
+
def test
|
60
|
+
raise AbstractMethodNotOverriddenError.new("PollCondition#test must be overridden in subclasses")
|
61
|
+
end
|
62
|
+
|
63
|
+
# Override this method in your Conditions (optional)
|
64
|
+
def after
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
class EventCondition < Condition
|
69
|
+
def register
|
70
|
+
raise AbstractMethodNotOverriddenError.new("EventCondition#register must be overridden in subclasses")
|
71
|
+
end
|
72
|
+
|
73
|
+
def deregister
|
74
|
+
raise AbstractMethodNotOverriddenError.new("EventCondition#deregister must be overridden in subclasses")
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
class TriggerCondition < Condition
|
79
|
+
def process(event, payload)
|
80
|
+
raise AbstractMethodNotOverriddenError.new("TriggerCondition#process must be overridden in subclasses")
|
81
|
+
end
|
82
|
+
|
83
|
+
def trigger
|
84
|
+
self.watch.trigger(self)
|
85
|
+
end
|
86
|
+
|
87
|
+
def register
|
88
|
+
Trigger.register(self)
|
89
|
+
end
|
90
|
+
|
91
|
+
def deregister
|
92
|
+
Trigger.deregister(self)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module God
|
2
|
+
module Conditions
|
3
|
+
|
4
|
+
class Always < PollCondition
|
5
|
+
attr_accessor :what
|
6
|
+
|
7
|
+
def initialize
|
8
|
+
self.info = "always"
|
9
|
+
end
|
10
|
+
|
11
|
+
def valid?
|
12
|
+
valid = true
|
13
|
+
valid &= complain("Attribute 'what' must be specified", self) if self.what.nil?
|
14
|
+
valid
|
15
|
+
end
|
16
|
+
|
17
|
+
def test
|
18
|
+
@what
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,86 @@
|
|
1
|
+
module God
|
2
|
+
module Conditions
|
3
|
+
|
4
|
+
class Complex < PollCondition
|
5
|
+
AND = 0x1
|
6
|
+
OR = 0x2
|
7
|
+
NOT = 0x4
|
8
|
+
|
9
|
+
def initialize()
|
10
|
+
super
|
11
|
+
|
12
|
+
@oper_stack = []
|
13
|
+
@op_stack = []
|
14
|
+
|
15
|
+
@this = nil
|
16
|
+
end
|
17
|
+
|
18
|
+
def valid?
|
19
|
+
@oper_stack.inject(true) { |acc, oper| acc & oper.valid? }
|
20
|
+
end
|
21
|
+
|
22
|
+
def prepare
|
23
|
+
@oper_stack.each { |oper| oper.prepare }
|
24
|
+
end
|
25
|
+
|
26
|
+
def new_oper(kind, op)
|
27
|
+
oper = Condition.generate(kind, self.watch)
|
28
|
+
@oper_stack.push(oper)
|
29
|
+
@op_stack.push(op)
|
30
|
+
oper
|
31
|
+
end
|
32
|
+
|
33
|
+
def this(kind)
|
34
|
+
@this = Condition.generate(kind, self.watch)
|
35
|
+
yield @this if block_given?
|
36
|
+
end
|
37
|
+
|
38
|
+
def and(kind)
|
39
|
+
oper = new_oper(kind, 0x1)
|
40
|
+
yield oper if block_given?
|
41
|
+
end
|
42
|
+
|
43
|
+
def and_not(kind)
|
44
|
+
oper = new_oper(kind, 0x5)
|
45
|
+
yield oper if block_given?
|
46
|
+
end
|
47
|
+
|
48
|
+
def or(kind)
|
49
|
+
oper = new_oper(kind, 0x2)
|
50
|
+
yield oper if block_given?
|
51
|
+
end
|
52
|
+
|
53
|
+
def or_not(kind)
|
54
|
+
oper = new_oper(kind, 0x6)
|
55
|
+
yield oper if block_given?
|
56
|
+
end
|
57
|
+
|
58
|
+
def test
|
59
|
+
if @this.nil?
|
60
|
+
# Although this() makes sense semantically and therefore
|
61
|
+
# encourages easy-to-read conditions, being able to omit it
|
62
|
+
# allows for more DRY code in some cases, so we deal with a
|
63
|
+
# nil @this here by initially setting res to true or false,
|
64
|
+
# depending on whether the first operator used is AND or OR
|
65
|
+
# respectively.
|
66
|
+
if 0 < @op_stack[0] & AND
|
67
|
+
res = true
|
68
|
+
else
|
69
|
+
res = false
|
70
|
+
end
|
71
|
+
else
|
72
|
+
res = @this.test
|
73
|
+
end
|
74
|
+
|
75
|
+
@op_stack.each do |op|
|
76
|
+
cond = @oper_stack.shift
|
77
|
+
eval "res " + ((0 < op & AND) ? "&&" : "||") + "= " + ((0 < op & NOT) ? "!" : "") + "cond.test"
|
78
|
+
@oper_stack.push cond
|
79
|
+
end
|
80
|
+
|
81
|
+
res
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
end
|
86
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
module God
|
2
|
+
module Conditions
|
3
|
+
|
4
|
+
# Condition Symbol :cpu_usage
|
5
|
+
# Type: Poll
|
6
|
+
#
|
7
|
+
# Trigger when the percent of CPU use of a process is above a specified limit.
|
8
|
+
# On multi-core systems, this number could conceivably be above 100.
|
9
|
+
#
|
10
|
+
# Paramaters
|
11
|
+
# Required
|
12
|
+
# +pid_file+ is the pid file of the process in question. Automatically
|
13
|
+
# populated for Watches.
|
14
|
+
# +above+ is the percent CPU above which to trigger the condition. You
|
15
|
+
# may use #percent to clarify this amount (see examples).
|
16
|
+
#
|
17
|
+
# Examples
|
18
|
+
#
|
19
|
+
# Trigger if the process is using more than 25 percent of the cpu (from a Watch):
|
20
|
+
#
|
21
|
+
# on.condition(:cpu_usage) do |c|
|
22
|
+
# c.above = 25.percent
|
23
|
+
# end
|
24
|
+
#
|
25
|
+
# Non-Watch Tasks must specify a PID file:
|
26
|
+
#
|
27
|
+
# on.condition(:cpu_usage) do |c|
|
28
|
+
# c.above = 25.percent
|
29
|
+
# c.pid_file = "/var/run/mongrel.3000.pid"
|
30
|
+
# end
|
31
|
+
class CpuUsage < PollCondition
|
32
|
+
attr_accessor :above, :times, :pid_file
|
33
|
+
|
34
|
+
def initialize
|
35
|
+
super
|
36
|
+
self.above = nil
|
37
|
+
self.times = [1, 1]
|
38
|
+
end
|
39
|
+
|
40
|
+
def prepare
|
41
|
+
if self.times.kind_of?(Integer)
|
42
|
+
self.times = [self.times, self.times]
|
43
|
+
end
|
44
|
+
|
45
|
+
@timeline = Timeline.new(self.times[1])
|
46
|
+
end
|
47
|
+
|
48
|
+
def reset
|
49
|
+
@timeline.clear
|
50
|
+
end
|
51
|
+
|
52
|
+
def pid
|
53
|
+
self.pid_file ? File.read(self.pid_file).strip.to_i : self.watch.pid
|
54
|
+
end
|
55
|
+
|
56
|
+
def valid?
|
57
|
+
valid = true
|
58
|
+
valid &= complain("Attribute 'pid_file' must be specified", self) if self.pid_file.nil? && self.watch.pid_file.nil?
|
59
|
+
valid &= complain("Attribute 'above' must be specified", self) if self.above.nil?
|
60
|
+
valid
|
61
|
+
end
|
62
|
+
|
63
|
+
def test
|
64
|
+
process = System::Process.new(self.pid)
|
65
|
+
@timeline.push(process.percent_cpu)
|
66
|
+
|
67
|
+
history = "[" + @timeline.map { |x| "#{x > self.above ? '*' : ''}#{x}%%" }.join(", ") + "]"
|
68
|
+
|
69
|
+
if @timeline.select { |x| x > self.above }.size >= self.times.first
|
70
|
+
self.info = "cpu out of bounds #{history}"
|
71
|
+
return true
|
72
|
+
else
|
73
|
+
self.info = "cpu within bounds #{history}"
|
74
|
+
return false
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
module God
|
2
|
+
module Conditions
|
3
|
+
|
4
|
+
# This condition degrades its interval by a factor of two for 3 tries before failing
|
5
|
+
class DegradingLambda < PollCondition
|
6
|
+
attr_accessor :lambda
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
super
|
10
|
+
@tries = 0
|
11
|
+
end
|
12
|
+
|
13
|
+
def valid?
|
14
|
+
valid = true
|
15
|
+
valid &= complain("Attribute 'lambda' must be specified", self) if self.lambda.nil?
|
16
|
+
valid
|
17
|
+
end
|
18
|
+
|
19
|
+
def test
|
20
|
+
puts "Calling test. Interval at #{self.interval}"
|
21
|
+
@original_interval ||= self.interval
|
22
|
+
unless pass?
|
23
|
+
if @tries == 2
|
24
|
+
self.info = "lambda condition was satisfied"
|
25
|
+
return true
|
26
|
+
end
|
27
|
+
self.interval = self.interval / 2.0
|
28
|
+
@tries += 1
|
29
|
+
else
|
30
|
+
@tries = 0
|
31
|
+
self.interval = @original_interval
|
32
|
+
end
|
33
|
+
|
34
|
+
self.info = "lambda condition was not satisfied"
|
35
|
+
false
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
|
40
|
+
def pass?
|
41
|
+
begin
|
42
|
+
Timeout::timeout(@interval) {
|
43
|
+
self.lambda.call()
|
44
|
+
}
|
45
|
+
rescue Timeout::Error
|
46
|
+
false
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module God
|
2
|
+
module Conditions
|
3
|
+
|
4
|
+
class DiskUsage < PollCondition
|
5
|
+
attr_accessor :above, :mount_point
|
6
|
+
|
7
|
+
def initialize
|
8
|
+
super
|
9
|
+
self.above = nil
|
10
|
+
self.mount_point = nil
|
11
|
+
end
|
12
|
+
|
13
|
+
def valid?
|
14
|
+
valid = true
|
15
|
+
valid &= complain("Attribute 'mount_point' must be specified", self) if self.mount_point.nil?
|
16
|
+
valid &= complain("Attribute 'above' must be specified", self) if self.above.nil?
|
17
|
+
valid
|
18
|
+
end
|
19
|
+
|
20
|
+
def test
|
21
|
+
usage = `df | grep -i " #{self.mount_point}$" | awk '{print $5}' | sed 's/%//'`
|
22
|
+
usage.to_i > self.above
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,128 @@
|
|
1
|
+
module God
|
2
|
+
module Conditions
|
3
|
+
|
4
|
+
# Condition Symbol :flapping
|
5
|
+
# Type: Trigger
|
6
|
+
#
|
7
|
+
# Trigger when a Task transitions to or from a state or states a given number
|
8
|
+
# of times within a given period.
|
9
|
+
#
|
10
|
+
# Paramaters
|
11
|
+
# Required
|
12
|
+
# +times+ is the number of times that the Task must transition before
|
13
|
+
# triggering.
|
14
|
+
# +within+ is the number of seconds within which the Task must transition
|
15
|
+
# the specified number of times before triggering. You may use
|
16
|
+
# the sugar methods #seconds, #minutes, #hours, #days to clarify
|
17
|
+
# your code (see examples).
|
18
|
+
# --one or both of--
|
19
|
+
# +from_state+ is the state (as a Symbol) from which the transition must occur.
|
20
|
+
# +to_state is the state (as a Symbol) to which the transition must occur.
|
21
|
+
#
|
22
|
+
# Optional:
|
23
|
+
# +retry_in+ is the number of seconds after which to re-monitor the Task after
|
24
|
+
# it has been disabled by the condition.
|
25
|
+
# +retry_times+ is the number of times after which to permanently unmonitor
|
26
|
+
# the Task.
|
27
|
+
# +retry_within+ is the number of seconds within which
|
28
|
+
#
|
29
|
+
# Examples
|
30
|
+
#
|
31
|
+
# Trigger if
|
32
|
+
class Flapping < TriggerCondition
|
33
|
+
attr_accessor :times,
|
34
|
+
:within,
|
35
|
+
:from_state,
|
36
|
+
:to_state,
|
37
|
+
:retry_in,
|
38
|
+
:retry_times,
|
39
|
+
:retry_within
|
40
|
+
|
41
|
+
def initialize
|
42
|
+
self.info = "process is flapping"
|
43
|
+
end
|
44
|
+
|
45
|
+
def prepare
|
46
|
+
@timeline = Timeline.new(self.times)
|
47
|
+
@retry_timeline = Timeline.new(self.retry_times)
|
48
|
+
end
|
49
|
+
|
50
|
+
def valid?
|
51
|
+
valid = true
|
52
|
+
valid &= complain("Attribute 'times' must be specified", self) if self.times.nil?
|
53
|
+
valid &= complain("Attribute 'within' must be specified", self) if self.within.nil?
|
54
|
+
valid &= complain("Attributes 'from_state', 'to_state', or both must be specified", self) if self.from_state.nil? && self.to_state.nil?
|
55
|
+
valid
|
56
|
+
end
|
57
|
+
|
58
|
+
def process(event, payload)
|
59
|
+
begin
|
60
|
+
if event == :state_change
|
61
|
+
event_from_state, event_to_state = *payload
|
62
|
+
|
63
|
+
from_state_match = !self.from_state || self.from_state && Array(self.from_state).include?(event_from_state)
|
64
|
+
to_state_match = !self.to_state || self.to_state && Array(self.to_state).include?(event_to_state)
|
65
|
+
|
66
|
+
if from_state_match && to_state_match
|
67
|
+
@timeline << Time.now
|
68
|
+
|
69
|
+
concensus = (@timeline.size == self.times)
|
70
|
+
duration = (@timeline.last - @timeline.first) < self.within
|
71
|
+
|
72
|
+
if concensus && duration
|
73
|
+
@timeline.clear
|
74
|
+
trigger
|
75
|
+
retry_mechanism
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
rescue => e
|
80
|
+
puts e.message
|
81
|
+
puts e.backtrace.join("\n")
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
private
|
86
|
+
|
87
|
+
def retry_mechanism
|
88
|
+
if self.retry_in
|
89
|
+
@retry_timeline << Time.now
|
90
|
+
|
91
|
+
concensus = (@retry_timeline.size == self.retry_times)
|
92
|
+
duration = (@retry_timeline.last - @retry_timeline.first) < self.retry_within
|
93
|
+
|
94
|
+
if concensus && duration
|
95
|
+
# give up
|
96
|
+
Thread.new do
|
97
|
+
sleep 1
|
98
|
+
|
99
|
+
# log
|
100
|
+
msg = "#{self.watch.name} giving up"
|
101
|
+
applog(self.watch, :info, msg)
|
102
|
+
end
|
103
|
+
else
|
104
|
+
# try again later
|
105
|
+
Thread.new do
|
106
|
+
sleep 1
|
107
|
+
|
108
|
+
# log
|
109
|
+
msg = "#{self.watch.name} auto-reenable monitoring in #{self.retry_in} seconds"
|
110
|
+
applog(self.watch, :info, msg)
|
111
|
+
|
112
|
+
sleep self.retry_in
|
113
|
+
|
114
|
+
# log
|
115
|
+
msg = "#{self.watch.name} auto-reenabling monitoring"
|
116
|
+
applog(self.watch, :info, msg)
|
117
|
+
|
118
|
+
if self.watch.state == :unmonitored
|
119
|
+
self.watch.monitor
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
end
|
128
|
+
end
|