mojombo-god 0.7.7
Sign up to get free protection for your applications and to get access to all the features.
- data/History.txt +255 -0
- data/Manifest.txt +107 -0
- data/README.txt +59 -0
- data/Rakefile +35 -0
- data/bin/god +127 -0
- data/examples/events.god +84 -0
- data/examples/gravatar.god +54 -0
- data/examples/single.god +66 -0
- data/ext/god/extconf.rb +55 -0
- data/ext/god/kqueue_handler.c +123 -0
- data/ext/god/netlink_handler.c +167 -0
- data/init/god +42 -0
- data/lib/god.rb +644 -0
- data/lib/god/behavior.rb +52 -0
- data/lib/god/behaviors/clean_pid_file.rb +21 -0
- data/lib/god/behaviors/clean_unix_socket.rb +21 -0
- data/lib/god/behaviors/notify_when_flapping.rb +51 -0
- data/lib/god/cli/command.rb +206 -0
- data/lib/god/cli/run.rb +177 -0
- data/lib/god/cli/version.rb +23 -0
- data/lib/god/condition.rb +96 -0
- data/lib/god/conditions/always.rb +23 -0
- data/lib/god/conditions/complex.rb +86 -0
- data/lib/god/conditions/cpu_usage.rb +80 -0
- data/lib/god/conditions/degrading_lambda.rb +52 -0
- data/lib/god/conditions/disk_usage.rb +27 -0
- data/lib/god/conditions/flapping.rb +128 -0
- data/lib/god/conditions/http_response_code.rb +168 -0
- data/lib/god/conditions/lambda.rb +25 -0
- data/lib/god/conditions/memory_usage.rb +82 -0
- data/lib/god/conditions/process_exits.rb +72 -0
- data/lib/god/conditions/process_running.rb +74 -0
- data/lib/god/conditions/tries.rb +44 -0
- data/lib/god/configurable.rb +57 -0
- data/lib/god/contact.rb +106 -0
- data/lib/god/contacts/email.rb +95 -0
- data/lib/god/dependency_graph.rb +41 -0
- data/lib/god/diagnostics.rb +37 -0
- data/lib/god/driver.rb +108 -0
- data/lib/god/errors.rb +24 -0
- data/lib/god/event_handler.rb +111 -0
- data/lib/god/event_handlers/dummy_handler.rb +13 -0
- data/lib/god/event_handlers/kqueue_handler.rb +17 -0
- data/lib/god/event_handlers/netlink_handler.rb +13 -0
- data/lib/god/logger.rb +120 -0
- data/lib/god/metric.rb +59 -0
- data/lib/god/process.rb +325 -0
- data/lib/god/registry.rb +32 -0
- data/lib/god/simple_logger.rb +53 -0
- data/lib/god/socket.rb +96 -0
- data/lib/god/sugar.rb +47 -0
- data/lib/god/system/portable_poller.rb +42 -0
- data/lib/god/system/process.rb +42 -0
- data/lib/god/system/slash_proc_poller.rb +82 -0
- data/lib/god/task.rb +487 -0
- data/lib/god/timeline.rb +25 -0
- data/lib/god/trigger.rb +43 -0
- data/lib/god/watch.rb +183 -0
- data/test/configs/child_events/child_events.god +44 -0
- data/test/configs/child_events/simple_server.rb +3 -0
- data/test/configs/child_polls/child_polls.god +37 -0
- data/test/configs/child_polls/simple_server.rb +12 -0
- data/test/configs/complex/complex.god +59 -0
- data/test/configs/complex/simple_server.rb +3 -0
- data/test/configs/contact/contact.god +74 -0
- data/test/configs/contact/simple_server.rb +3 -0
- data/test/configs/daemon_events/daemon_events.god +37 -0
- data/test/configs/daemon_events/simple_server.rb +8 -0
- data/test/configs/daemon_events/simple_server_stop.rb +11 -0
- data/test/configs/daemon_polls/daemon_polls.god +17 -0
- data/test/configs/daemon_polls/simple_server.rb +6 -0
- data/test/configs/degrading_lambda/degrading_lambda.god +31 -0
- data/test/configs/degrading_lambda/tcp_server.rb +15 -0
- data/test/configs/matias/matias.god +50 -0
- data/test/configs/real.rb +59 -0
- data/test/configs/running_load/running_load.god +16 -0
- data/test/configs/stress/simple_server.rb +3 -0
- data/test/configs/stress/stress.god +15 -0
- data/test/configs/task/logs/.placeholder +0 -0
- data/test/configs/task/task.god +26 -0
- data/test/configs/test.rb +61 -0
- data/test/helper.rb +151 -0
- data/test/suite.rb +6 -0
- data/test/test_behavior.rb +21 -0
- data/test/test_condition.rb +50 -0
- data/test/test_conditions_disk_usage.rb +56 -0
- data/test/test_conditions_http_response_code.rb +109 -0
- data/test/test_conditions_process_running.rb +44 -0
- data/test/test_conditions_tries.rb +67 -0
- data/test/test_contact.rb +109 -0
- data/test/test_dependency_graph.rb +62 -0
- data/test/test_driver.rb +11 -0
- data/test/test_event_handler.rb +80 -0
- data/test/test_god.rb +598 -0
- data/test/test_handlers_kqueue_handler.rb +16 -0
- data/test/test_logger.rb +63 -0
- data/test/test_metric.rb +72 -0
- data/test/test_process.rb +246 -0
- data/test/test_registry.rb +15 -0
- data/test/test_socket.rb +42 -0
- data/test/test_sugar.rb +42 -0
- data/test/test_system_portable_poller.rb +17 -0
- data/test/test_system_process.rb +30 -0
- data/test/test_task.rb +262 -0
- data/test/test_timeline.rb +37 -0
- data/test/test_trigger.rb +59 -0
- data/test/test_watch.rb +279 -0
- metadata +186 -0
@@ -0,0 +1,23 @@
|
|
1
|
+
module God
|
2
|
+
module CLI
|
3
|
+
|
4
|
+
class Version
|
5
|
+
def self.version
|
6
|
+
require 'god'
|
7
|
+
|
8
|
+
# print version
|
9
|
+
puts "Version #{God::VERSION}"
|
10
|
+
exit
|
11
|
+
end
|
12
|
+
|
13
|
+
def self.version_extended
|
14
|
+
puts "Version: #{God::VERSION}"
|
15
|
+
puts "Polls: enabled"
|
16
|
+
puts "Events: " + God::EventHandler.event_system
|
17
|
+
|
18
|
+
exit
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
module God
|
2
|
+
|
3
|
+
class Condition < Behavior
|
4
|
+
attr_accessor :transition, :notify, :info, :phase
|
5
|
+
|
6
|
+
# Generate a Condition of the given kind. The proper class if found by camel casing the
|
7
|
+
# kind (which is given as an underscored symbol).
|
8
|
+
# +kind+ is the underscored symbol representing the class (e.g. :foo_bar for God::Conditions::FooBar)
|
9
|
+
def self.generate(kind, watch)
|
10
|
+
sym = kind.to_s.capitalize.gsub(/_(.)/){$1.upcase}.intern
|
11
|
+
c = God::Conditions.const_get(sym).new
|
12
|
+
|
13
|
+
unless c.kind_of?(PollCondition) || c.kind_of?(EventCondition) || c.kind_of?(TriggerCondition)
|
14
|
+
abort "Condition '#{c.class.name}' must subclass God::PollCondition, God::EventCondition, or God::TriggerCondition"
|
15
|
+
end
|
16
|
+
|
17
|
+
if !EventHandler.loaded? && c.kind_of?(EventCondition)
|
18
|
+
abort "Condition '#{c.class.name}' requires an event system but none has been loaded"
|
19
|
+
end
|
20
|
+
|
21
|
+
c.watch = watch
|
22
|
+
c
|
23
|
+
rescue NameError
|
24
|
+
raise NoSuchConditionError.new("No Condition found with the class name God::Conditions::#{sym}")
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.valid?(condition)
|
28
|
+
valid = true
|
29
|
+
if condition.notify
|
30
|
+
begin
|
31
|
+
Contact.normalize(condition.notify)
|
32
|
+
rescue ArgumentError => e
|
33
|
+
valid &= Configurable.complain("Attribute 'notify' " + e.message, condition)
|
34
|
+
end
|
35
|
+
end
|
36
|
+
valid
|
37
|
+
end
|
38
|
+
|
39
|
+
# Construct the friendly name of this Condition, looks like:
|
40
|
+
#
|
41
|
+
# Condition FooBar on Watch 'baz'
|
42
|
+
def friendly_name
|
43
|
+
"Condition #{self.class.name.split('::').last} on Watch '#{self.watch.name}'"
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
47
|
+
class PollCondition < Condition
|
48
|
+
# all poll conditions can specify a poll interval
|
49
|
+
attr_accessor :interval
|
50
|
+
|
51
|
+
# Override this method in your Conditions (optional)
|
52
|
+
def before
|
53
|
+
end
|
54
|
+
|
55
|
+
# Override this method in your Conditions (mandatory)
|
56
|
+
#
|
57
|
+
# Return true if the test passes (everything is ok)
|
58
|
+
# Return false otherwise
|
59
|
+
def test
|
60
|
+
raise AbstractMethodNotOverriddenError.new("PollCondition#test must be overridden in subclasses")
|
61
|
+
end
|
62
|
+
|
63
|
+
# Override this method in your Conditions (optional)
|
64
|
+
def after
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
class EventCondition < Condition
|
69
|
+
def register
|
70
|
+
raise AbstractMethodNotOverriddenError.new("EventCondition#register must be overridden in subclasses")
|
71
|
+
end
|
72
|
+
|
73
|
+
def deregister
|
74
|
+
raise AbstractMethodNotOverriddenError.new("EventCondition#deregister must be overridden in subclasses")
|
75
|
+
end
|
76
|
+
end
|
77
|
+
|
78
|
+
class TriggerCondition < Condition
|
79
|
+
def process(event, payload)
|
80
|
+
raise AbstractMethodNotOverriddenError.new("TriggerCondition#process must be overridden in subclasses")
|
81
|
+
end
|
82
|
+
|
83
|
+
def trigger
|
84
|
+
self.watch.trigger(self)
|
85
|
+
end
|
86
|
+
|
87
|
+
def register
|
88
|
+
Trigger.register(self)
|
89
|
+
end
|
90
|
+
|
91
|
+
def deregister
|
92
|
+
Trigger.deregister(self)
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
end
|
@@ -0,0 +1,23 @@
|
|
1
|
+
module God
|
2
|
+
module Conditions
|
3
|
+
|
4
|
+
class Always < PollCondition
|
5
|
+
attr_accessor :what
|
6
|
+
|
7
|
+
def initialize
|
8
|
+
self.info = "always"
|
9
|
+
end
|
10
|
+
|
11
|
+
def valid?
|
12
|
+
valid = true
|
13
|
+
valid &= complain("Attribute 'what' must be specified", self) if self.what.nil?
|
14
|
+
valid
|
15
|
+
end
|
16
|
+
|
17
|
+
def test
|
18
|
+
@what
|
19
|
+
end
|
20
|
+
end
|
21
|
+
|
22
|
+
end
|
23
|
+
end
|
@@ -0,0 +1,86 @@
|
|
1
|
+
module God
|
2
|
+
module Conditions
|
3
|
+
|
4
|
+
class Complex < PollCondition
|
5
|
+
AND = 0x1
|
6
|
+
OR = 0x2
|
7
|
+
NOT = 0x4
|
8
|
+
|
9
|
+
def initialize()
|
10
|
+
super
|
11
|
+
|
12
|
+
@oper_stack = []
|
13
|
+
@op_stack = []
|
14
|
+
|
15
|
+
@this = nil
|
16
|
+
end
|
17
|
+
|
18
|
+
def valid?
|
19
|
+
@oper_stack.inject(true) { |acc, oper| acc & oper.valid? }
|
20
|
+
end
|
21
|
+
|
22
|
+
def prepare
|
23
|
+
@oper_stack.each { |oper| oper.prepare }
|
24
|
+
end
|
25
|
+
|
26
|
+
def new_oper(kind, op)
|
27
|
+
oper = Condition.generate(kind, self.watch)
|
28
|
+
@oper_stack.push(oper)
|
29
|
+
@op_stack.push(op)
|
30
|
+
oper
|
31
|
+
end
|
32
|
+
|
33
|
+
def this(kind)
|
34
|
+
@this = Condition.generate(kind, self.watch)
|
35
|
+
yield @this if block_given?
|
36
|
+
end
|
37
|
+
|
38
|
+
def and(kind)
|
39
|
+
oper = new_oper(kind, 0x1)
|
40
|
+
yield oper if block_given?
|
41
|
+
end
|
42
|
+
|
43
|
+
def and_not(kind)
|
44
|
+
oper = new_oper(kind, 0x5)
|
45
|
+
yield oper if block_given?
|
46
|
+
end
|
47
|
+
|
48
|
+
def or(kind)
|
49
|
+
oper = new_oper(kind, 0x2)
|
50
|
+
yield oper if block_given?
|
51
|
+
end
|
52
|
+
|
53
|
+
def or_not(kind)
|
54
|
+
oper = new_oper(kind, 0x6)
|
55
|
+
yield oper if block_given?
|
56
|
+
end
|
57
|
+
|
58
|
+
def test
|
59
|
+
if @this.nil?
|
60
|
+
# Although this() makes sense semantically and therefore
|
61
|
+
# encourages easy-to-read conditions, being able to omit it
|
62
|
+
# allows for more DRY code in some cases, so we deal with a
|
63
|
+
# nil @this here by initially setting res to true or false,
|
64
|
+
# depending on whether the first operator used is AND or OR
|
65
|
+
# respectively.
|
66
|
+
if 0 < @op_stack[0] & AND
|
67
|
+
res = true
|
68
|
+
else
|
69
|
+
res = false
|
70
|
+
end
|
71
|
+
else
|
72
|
+
res = @this.test
|
73
|
+
end
|
74
|
+
|
75
|
+
@op_stack.each do |op|
|
76
|
+
cond = @oper_stack.shift
|
77
|
+
eval "res " + ((0 < op & AND) ? "&&" : "||") + "= " + ((0 < op & NOT) ? "!" : "") + "cond.test"
|
78
|
+
@oper_stack.push cond
|
79
|
+
end
|
80
|
+
|
81
|
+
res
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
end
|
86
|
+
end
|
@@ -0,0 +1,80 @@
|
|
1
|
+
module God
|
2
|
+
module Conditions
|
3
|
+
|
4
|
+
# Condition Symbol :cpu_usage
|
5
|
+
# Type: Poll
|
6
|
+
#
|
7
|
+
# Trigger when the percent of CPU use of a process is above a specified limit.
|
8
|
+
# On multi-core systems, this number could conceivably be above 100.
|
9
|
+
#
|
10
|
+
# Paramaters
|
11
|
+
# Required
|
12
|
+
# +pid_file+ is the pid file of the process in question. Automatically
|
13
|
+
# populated for Watches.
|
14
|
+
# +above+ is the percent CPU above which to trigger the condition. You
|
15
|
+
# may use #percent to clarify this amount (see examples).
|
16
|
+
#
|
17
|
+
# Examples
|
18
|
+
#
|
19
|
+
# Trigger if the process is using more than 25 percent of the cpu (from a Watch):
|
20
|
+
#
|
21
|
+
# on.condition(:cpu_usage) do |c|
|
22
|
+
# c.above = 25.percent
|
23
|
+
# end
|
24
|
+
#
|
25
|
+
# Non-Watch Tasks must specify a PID file:
|
26
|
+
#
|
27
|
+
# on.condition(:cpu_usage) do |c|
|
28
|
+
# c.above = 25.percent
|
29
|
+
# c.pid_file = "/var/run/mongrel.3000.pid"
|
30
|
+
# end
|
31
|
+
class CpuUsage < PollCondition
|
32
|
+
attr_accessor :above, :times, :pid_file
|
33
|
+
|
34
|
+
def initialize
|
35
|
+
super
|
36
|
+
self.above = nil
|
37
|
+
self.times = [1, 1]
|
38
|
+
end
|
39
|
+
|
40
|
+
def prepare
|
41
|
+
if self.times.kind_of?(Integer)
|
42
|
+
self.times = [self.times, self.times]
|
43
|
+
end
|
44
|
+
|
45
|
+
@timeline = Timeline.new(self.times[1])
|
46
|
+
end
|
47
|
+
|
48
|
+
def reset
|
49
|
+
@timeline.clear
|
50
|
+
end
|
51
|
+
|
52
|
+
def pid
|
53
|
+
self.pid_file ? File.read(self.pid_file).strip.to_i : self.watch.pid
|
54
|
+
end
|
55
|
+
|
56
|
+
def valid?
|
57
|
+
valid = true
|
58
|
+
valid &= complain("Attribute 'pid_file' must be specified", self) if self.pid_file.nil? && self.watch.pid_file.nil?
|
59
|
+
valid &= complain("Attribute 'above' must be specified", self) if self.above.nil?
|
60
|
+
valid
|
61
|
+
end
|
62
|
+
|
63
|
+
def test
|
64
|
+
process = System::Process.new(self.pid)
|
65
|
+
@timeline.push(process.percent_cpu)
|
66
|
+
|
67
|
+
history = "[" + @timeline.map { |x| "#{x > self.above ? '*' : ''}#{x}%%" }.join(", ") + "]"
|
68
|
+
|
69
|
+
if @timeline.select { |x| x > self.above }.size >= self.times.first
|
70
|
+
self.info = "cpu out of bounds #{history}"
|
71
|
+
return true
|
72
|
+
else
|
73
|
+
self.info = "cpu within bounds #{history}"
|
74
|
+
return false
|
75
|
+
end
|
76
|
+
end
|
77
|
+
end
|
78
|
+
|
79
|
+
end
|
80
|
+
end
|
@@ -0,0 +1,52 @@
|
|
1
|
+
module God
|
2
|
+
module Conditions
|
3
|
+
|
4
|
+
# This condition degrades its interval by a factor of two for 3 tries before failing
|
5
|
+
class DegradingLambda < PollCondition
|
6
|
+
attr_accessor :lambda
|
7
|
+
|
8
|
+
def initialize
|
9
|
+
super
|
10
|
+
@tries = 0
|
11
|
+
end
|
12
|
+
|
13
|
+
def valid?
|
14
|
+
valid = true
|
15
|
+
valid &= complain("Attribute 'lambda' must be specified", self) if self.lambda.nil?
|
16
|
+
valid
|
17
|
+
end
|
18
|
+
|
19
|
+
def test
|
20
|
+
puts "Calling test. Interval at #{self.interval}"
|
21
|
+
@original_interval ||= self.interval
|
22
|
+
unless pass?
|
23
|
+
if @tries == 2
|
24
|
+
self.info = "lambda condition was satisfied"
|
25
|
+
return true
|
26
|
+
end
|
27
|
+
self.interval = self.interval / 2.0
|
28
|
+
@tries += 1
|
29
|
+
else
|
30
|
+
@tries = 0
|
31
|
+
self.interval = @original_interval
|
32
|
+
end
|
33
|
+
|
34
|
+
self.info = "lambda condition was not satisfied"
|
35
|
+
false
|
36
|
+
end
|
37
|
+
|
38
|
+
private
|
39
|
+
|
40
|
+
def pass?
|
41
|
+
begin
|
42
|
+
Timeout::timeout(@interval) {
|
43
|
+
self.lambda.call()
|
44
|
+
}
|
45
|
+
rescue Timeout::Error
|
46
|
+
false
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
50
|
+
|
51
|
+
end
|
52
|
+
end
|
@@ -0,0 +1,27 @@
|
|
1
|
+
module God
|
2
|
+
module Conditions
|
3
|
+
|
4
|
+
class DiskUsage < PollCondition
|
5
|
+
attr_accessor :above, :mount_point
|
6
|
+
|
7
|
+
def initialize
|
8
|
+
super
|
9
|
+
self.above = nil
|
10
|
+
self.mount_point = nil
|
11
|
+
end
|
12
|
+
|
13
|
+
def valid?
|
14
|
+
valid = true
|
15
|
+
valid &= complain("Attribute 'mount_point' must be specified", self) if self.mount_point.nil?
|
16
|
+
valid &= complain("Attribute 'above' must be specified", self) if self.above.nil?
|
17
|
+
valid
|
18
|
+
end
|
19
|
+
|
20
|
+
def test
|
21
|
+
usage = `df | grep -i " #{self.mount_point}$" | awk '{print $5}' | sed 's/%//'`
|
22
|
+
usage.to_i > self.above
|
23
|
+
end
|
24
|
+
end
|
25
|
+
|
26
|
+
end
|
27
|
+
end
|
@@ -0,0 +1,128 @@
|
|
1
|
+
module God
|
2
|
+
module Conditions
|
3
|
+
|
4
|
+
# Condition Symbol :flapping
|
5
|
+
# Type: Trigger
|
6
|
+
#
|
7
|
+
# Trigger when a Task transitions to or from a state or states a given number
|
8
|
+
# of times within a given period.
|
9
|
+
#
|
10
|
+
# Paramaters
|
11
|
+
# Required
|
12
|
+
# +times+ is the number of times that the Task must transition before
|
13
|
+
# triggering.
|
14
|
+
# +within+ is the number of seconds within which the Task must transition
|
15
|
+
# the specified number of times before triggering. You may use
|
16
|
+
# the sugar methods #seconds, #minutes, #hours, #days to clarify
|
17
|
+
# your code (see examples).
|
18
|
+
# --one or both of--
|
19
|
+
# +from_state+ is the state (as a Symbol) from which the transition must occur.
|
20
|
+
# +to_state is the state (as a Symbol) to which the transition must occur.
|
21
|
+
#
|
22
|
+
# Optional:
|
23
|
+
# +retry_in+ is the number of seconds after which to re-monitor the Task after
|
24
|
+
# it has been disabled by the condition.
|
25
|
+
# +retry_times+ is the number of times after which to permanently unmonitor
|
26
|
+
# the Task.
|
27
|
+
# +retry_within+ is the number of seconds within which
|
28
|
+
#
|
29
|
+
# Examples
|
30
|
+
#
|
31
|
+
# Trigger if
|
32
|
+
class Flapping < TriggerCondition
|
33
|
+
attr_accessor :times,
|
34
|
+
:within,
|
35
|
+
:from_state,
|
36
|
+
:to_state,
|
37
|
+
:retry_in,
|
38
|
+
:retry_times,
|
39
|
+
:retry_within
|
40
|
+
|
41
|
+
def initialize
|
42
|
+
self.info = "process is flapping"
|
43
|
+
end
|
44
|
+
|
45
|
+
def prepare
|
46
|
+
@timeline = Timeline.new(self.times)
|
47
|
+
@retry_timeline = Timeline.new(self.retry_times)
|
48
|
+
end
|
49
|
+
|
50
|
+
def valid?
|
51
|
+
valid = true
|
52
|
+
valid &= complain("Attribute 'times' must be specified", self) if self.times.nil?
|
53
|
+
valid &= complain("Attribute 'within' must be specified", self) if self.within.nil?
|
54
|
+
valid &= complain("Attributes 'from_state', 'to_state', or both must be specified", self) if self.from_state.nil? && self.to_state.nil?
|
55
|
+
valid
|
56
|
+
end
|
57
|
+
|
58
|
+
def process(event, payload)
|
59
|
+
begin
|
60
|
+
if event == :state_change
|
61
|
+
event_from_state, event_to_state = *payload
|
62
|
+
|
63
|
+
from_state_match = !self.from_state || self.from_state && Array(self.from_state).include?(event_from_state)
|
64
|
+
to_state_match = !self.to_state || self.to_state && Array(self.to_state).include?(event_to_state)
|
65
|
+
|
66
|
+
if from_state_match && to_state_match
|
67
|
+
@timeline << Time.now
|
68
|
+
|
69
|
+
concensus = (@timeline.size == self.times)
|
70
|
+
duration = (@timeline.last - @timeline.first) < self.within
|
71
|
+
|
72
|
+
if concensus && duration
|
73
|
+
@timeline.clear
|
74
|
+
trigger
|
75
|
+
retry_mechanism
|
76
|
+
end
|
77
|
+
end
|
78
|
+
end
|
79
|
+
rescue => e
|
80
|
+
puts e.message
|
81
|
+
puts e.backtrace.join("\n")
|
82
|
+
end
|
83
|
+
end
|
84
|
+
|
85
|
+
private
|
86
|
+
|
87
|
+
def retry_mechanism
|
88
|
+
if self.retry_in
|
89
|
+
@retry_timeline << Time.now
|
90
|
+
|
91
|
+
concensus = (@retry_timeline.size == self.retry_times)
|
92
|
+
duration = (@retry_timeline.last - @retry_timeline.first) < self.retry_within
|
93
|
+
|
94
|
+
if concensus && duration
|
95
|
+
# give up
|
96
|
+
Thread.new do
|
97
|
+
sleep 1
|
98
|
+
|
99
|
+
# log
|
100
|
+
msg = "#{self.watch.name} giving up"
|
101
|
+
applog(self.watch, :info, msg)
|
102
|
+
end
|
103
|
+
else
|
104
|
+
# try again later
|
105
|
+
Thread.new do
|
106
|
+
sleep 1
|
107
|
+
|
108
|
+
# log
|
109
|
+
msg = "#{self.watch.name} auto-reenable monitoring in #{self.retry_in} seconds"
|
110
|
+
applog(self.watch, :info, msg)
|
111
|
+
|
112
|
+
sleep self.retry_in
|
113
|
+
|
114
|
+
# log
|
115
|
+
msg = "#{self.watch.name} auto-reenabling monitoring"
|
116
|
+
applog(self.watch, :info, msg)
|
117
|
+
|
118
|
+
if self.watch.state == :unmonitored
|
119
|
+
self.watch.monitor
|
120
|
+
end
|
121
|
+
end
|
122
|
+
end
|
123
|
+
end
|
124
|
+
end
|
125
|
+
end
|
126
|
+
|
127
|
+
end
|
128
|
+
end
|