god 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +9 -1
- data/Manifest.txt +13 -1
- data/README.txt +2 -0
- data/Rakefile +2 -2
- data/examples/gravatar.god +8 -2
- data/examples/local.god +4 -4
- data/ext/god/Makefile +149 -0
- data/ext/god/extconf.rb +8 -0
- data/ext/god/kqueue_handler.c +122 -0
- data/ext/god/netlink_handler.c +140 -0
- data/lib/god.rb +37 -4
- data/lib/god/conditions/always.rb +9 -1
- data/lib/god/conditions/cpu_usage.rb +2 -2
- data/lib/god/conditions/memory_usage.rb +2 -2
- data/lib/god/conditions/process_exits.rb +28 -0
- data/lib/god/conditions/process_running.rb +25 -0
- data/lib/god/event_handler.rb +45 -0
- data/lib/god/event_handlers/kqueue_handler.rb +15 -0
- data/lib/god/event_handlers/netlink_handler.rb +11 -0
- data/lib/god/hub.rb +84 -0
- data/lib/god/meddle.rb +1 -17
- data/lib/god/metric.rb +60 -0
- data/lib/god/timer.rb +21 -26
- data/lib/god/watch.rb +90 -58
- data/test/configs/real.rb +2 -5
- data/test/configs/test.rb +75 -0
- data/test/helper.rb +1 -0
- data/test/test_condition.rb +1 -1
- data/test/test_god.rb +3 -2
- data/test/test_metric.rb +60 -0
- data/test/test_timer.rb +21 -11
- data/test/test_watch.rb +49 -78
- metadata +20 -6
- data/lib/god/conditions/process_not_running.rb +0 -22
data/lib/god/watch.rb
CHANGED
@@ -1,11 +1,13 @@
|
|
1
1
|
module God
|
2
2
|
|
3
3
|
class Watch < Base
|
4
|
+
VALID_STATES = [:init, :up, :start, :restart]
|
5
|
+
|
4
6
|
# config
|
5
|
-
attr_accessor :name, :start, :stop, :restart, :interval, :grace
|
7
|
+
attr_accessor :name, :state, :start, :stop, :restart, :interval, :grace
|
6
8
|
|
7
9
|
# api
|
8
|
-
attr_accessor :behaviors, :
|
10
|
+
attr_accessor :behaviors, :metrics
|
9
11
|
|
10
12
|
# internal
|
11
13
|
attr_accessor :mutex
|
@@ -13,18 +15,18 @@ module God
|
|
13
15
|
#
|
14
16
|
def initialize(meddle)
|
15
17
|
@meddle = meddle
|
16
|
-
|
18
|
+
|
17
19
|
# no grace period by default
|
18
20
|
self.grace = 0
|
19
21
|
|
20
|
-
#
|
21
|
-
@action = nil
|
22
|
-
|
22
|
+
# the list of behaviors
|
23
23
|
self.behaviors = []
|
24
24
|
|
25
25
|
# the list of conditions for each action
|
26
|
-
self.
|
27
|
-
|
26
|
+
self.metrics = {:init => [],
|
27
|
+
:start => [],
|
28
|
+
:restart => [],
|
29
|
+
:up => []}
|
28
30
|
|
29
31
|
# mutex
|
30
32
|
self.mutex = Mutex.new
|
@@ -48,67 +50,89 @@ module God
|
|
48
50
|
self.behaviors << b
|
49
51
|
end
|
50
52
|
|
53
|
+
###########################################################################
|
54
|
+
#
|
55
|
+
# Advanced mode
|
56
|
+
#
|
57
|
+
###########################################################################
|
58
|
+
|
59
|
+
# Define a transition handler which consists of a set of conditions
|
60
|
+
def transition(start_states, end_states)
|
61
|
+
# convert to into canonical hash form
|
62
|
+
canonical_end_states = canonical_hash_form(end_states)
|
63
|
+
|
64
|
+
# for each start state do
|
65
|
+
Array(start_states).each do |start_state|
|
66
|
+
# validate start state
|
67
|
+
unless VALID_STATES.include?(start_state)
|
68
|
+
abort "Invalid state :#{start_state}. Must be one of the symbols #{VALID_STATES.map{|x| ":#{x}"}.join(', ')}"
|
69
|
+
end
|
70
|
+
|
71
|
+
# create a new metric to hold the watch, end states, and conditions
|
72
|
+
m = Metric.new(self, canonical_end_states)
|
73
|
+
|
74
|
+
# let the config file define some conditions on the metric
|
75
|
+
yield(m)
|
76
|
+
|
77
|
+
# record the metric
|
78
|
+
self.metrics[start_state] << m
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
###########################################################################
|
83
|
+
#
|
84
|
+
# Simple mode
|
85
|
+
#
|
86
|
+
###########################################################################
|
87
|
+
|
51
88
|
def start_if
|
52
|
-
|
53
|
-
|
54
|
-
|
89
|
+
self.transition(:up, :start) do |on|
|
90
|
+
yield(on)
|
91
|
+
end
|
55
92
|
end
|
56
93
|
|
57
94
|
def restart_if
|
58
|
-
|
59
|
-
|
60
|
-
|
95
|
+
self.transition(:up, :restart) do |on|
|
96
|
+
yield(on)
|
97
|
+
end
|
61
98
|
end
|
62
99
|
|
63
|
-
|
64
|
-
#
|
65
|
-
|
66
|
-
|
67
|
-
|
68
|
-
|
100
|
+
###########################################################################
|
101
|
+
#
|
102
|
+
# Lifecycle
|
103
|
+
#
|
104
|
+
###########################################################################
|
105
|
+
|
106
|
+
# Schedule all poll conditions and register all condition events
|
107
|
+
def monitor
|
108
|
+
# start monitoring at the first available of the init or up states
|
109
|
+
if !self.metrics[:init].empty?
|
110
|
+
self.move(:init)
|
111
|
+
else
|
112
|
+
self.move(:up)
|
69
113
|
end
|
70
|
-
|
71
|
-
|
72
|
-
|
73
|
-
|
74
|
-
|
75
|
-
|
114
|
+
end
|
115
|
+
|
116
|
+
# Move from one state to another
|
117
|
+
def move(to_state)
|
118
|
+
puts "move '#{self.state}' to '#{to_state}'"
|
119
|
+
|
120
|
+
# cleanup from current state
|
121
|
+
if from_state = self.state
|
122
|
+
self.metrics[from_state].each { |m| m.disable }
|
76
123
|
end
|
77
124
|
|
78
|
-
#
|
79
|
-
|
80
|
-
|
81
|
-
# call prepare on the condition
|
82
|
-
c.prepare
|
125
|
+
# perform action (if available)
|
126
|
+
self.action(to_state)
|
83
127
|
|
84
|
-
#
|
85
|
-
|
86
|
-
unless c.valid?
|
87
|
-
abort
|
88
|
-
end
|
89
|
-
|
90
|
-
# inherit interval from meddle if no poll condition specific interval was set
|
91
|
-
if c.kind_of?(PollCondition) && !c.interval
|
92
|
-
if self.interval
|
93
|
-
c.interval = self.interval
|
94
|
-
else
|
95
|
-
abort "No interval set for Condition '#{c.class.name}' in Watch '#{self.name}', and no default Watch interval from which to inherit"
|
96
|
-
end
|
97
|
-
end
|
128
|
+
# move to new state
|
129
|
+
self.metrics[to_state].each { |m| m.enable }
|
98
130
|
|
99
|
-
|
100
|
-
|
101
|
-
|
102
|
-
# Schedule all poll conditions and register all condition events
|
103
|
-
def monitor
|
104
|
-
[:start, :restart].each do |cmd|
|
105
|
-
self.conditions[cmd].each do |c|
|
106
|
-
@meddle.timer.register(self, c, cmd) if c.kind_of?(PollCondition)
|
107
|
-
end
|
108
|
-
end
|
131
|
+
# set state
|
132
|
+
self.state = to_state
|
109
133
|
end
|
110
134
|
|
111
|
-
def action(a, c)
|
135
|
+
def action(a, c = nil)
|
112
136
|
case a
|
113
137
|
when :start
|
114
138
|
puts self.start
|
@@ -132,7 +156,9 @@ module God
|
|
132
156
|
|
133
157
|
def call_action(condition, action, command)
|
134
158
|
# before
|
135
|
-
|
159
|
+
before_items = self.behaviors
|
160
|
+
before_items += [condition] if condition
|
161
|
+
before_items.each { |b| b.send("before_#{action}") }
|
136
162
|
|
137
163
|
# action
|
138
164
|
if command.kind_of?(String)
|
@@ -144,7 +170,13 @@ module God
|
|
144
170
|
end
|
145
171
|
|
146
172
|
# after
|
147
|
-
|
173
|
+
after_items = self.behaviors
|
174
|
+
after_items += [condition] if condition
|
175
|
+
after_items.each { |b| b.send("after_#{action}") }
|
176
|
+
end
|
177
|
+
|
178
|
+
def canonical_hash_form(to)
|
179
|
+
to.instance_of?(Symbol) ? {true => to} : to
|
148
180
|
end
|
149
181
|
end
|
150
182
|
|
data/test/configs/real.rb
CHANGED
@@ -21,11 +21,8 @@ God.meddle do |god|
|
|
21
21
|
|
22
22
|
# start if process is not running
|
23
23
|
w.start_if do |start|
|
24
|
-
|
25
|
-
|
26
|
-
# end
|
27
|
-
|
28
|
-
start.condition(:process_not_running) do |c|
|
24
|
+
start.condition(:process_running) do |c|
|
25
|
+
c.running = false
|
29
26
|
c.pid_file = pid_file
|
30
27
|
end
|
31
28
|
end
|
@@ -0,0 +1,75 @@
|
|
1
|
+
if $0 == __FILE__
|
2
|
+
require File.join(File.dirname(__FILE__), *%w[.. .. lib god])
|
3
|
+
end
|
4
|
+
|
5
|
+
RAILS_ROOT = "/Users/tom/dev/git/helloworld"
|
6
|
+
|
7
|
+
God.meddle do |god|
|
8
|
+
god.watch do |w|
|
9
|
+
w.name = "local-3000"
|
10
|
+
w.interval = 5 # seconds
|
11
|
+
w.start = "mongrel_rails start -P ./log/mongrel.pid -c #{RAILS_ROOT} -p 3001 -d"
|
12
|
+
w.stop = "mongrel_rails stop -P ./log/mongrel.pid -c #{RAILS_ROOT}"
|
13
|
+
|
14
|
+
pid_file = File.join(RAILS_ROOT, "log/mongrel.pid")
|
15
|
+
|
16
|
+
# clean pid files before start if necessary
|
17
|
+
w.behavior(:clean_pid_file) do |b|
|
18
|
+
b.pid_file = pid_file
|
19
|
+
end
|
20
|
+
|
21
|
+
# determine the state on startup
|
22
|
+
w.transition(:init, { true => :up, false => :start }) do |on|
|
23
|
+
on.condition(:process_running) do |c|
|
24
|
+
c.running = true
|
25
|
+
c.pid_file = pid_file
|
26
|
+
end
|
27
|
+
end
|
28
|
+
|
29
|
+
# determine when process has finished starting
|
30
|
+
w.transition([:start, :restart], :up) do |on|
|
31
|
+
on.condition(:process_running) do |c|
|
32
|
+
c.running = true
|
33
|
+
c.pid_file = pid_file
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
# start if process is not running
|
38
|
+
w.transition(:up, :start) do |on|
|
39
|
+
on.condition(:process_exits) do |c|
|
40
|
+
c.pid_file = pid_file
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
# restart if memory or cpu is too high
|
45
|
+
w.transition(:up, :restart) do |on|
|
46
|
+
on.condition(:memory_usage) do |c|
|
47
|
+
c.interval = 20
|
48
|
+
c.pid_file = pid_file
|
49
|
+
c.above = (50 * 1024) # 50mb
|
50
|
+
c.times = [3, 5]
|
51
|
+
end
|
52
|
+
|
53
|
+
on.condition(:cpu_usage) do |c|
|
54
|
+
c.interval = 10
|
55
|
+
c.pid_file = pid_file
|
56
|
+
c.above = 10 # percent
|
57
|
+
c.times = [3, 5]
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
61
|
+
|
62
|
+
# clear old session files
|
63
|
+
# god.watch do |w|
|
64
|
+
# w.name = "local-session-cleanup"
|
65
|
+
# w.start = lambda do
|
66
|
+
# Dir["#{RAILS_ROOT}/tmp/sessions/ruby_sess.*"].select do |f|
|
67
|
+
# File.mtime(f) < Time.now - (7 * 24 * 60 * 60)
|
68
|
+
# end.each { |f| File.delete(f) }
|
69
|
+
# end
|
70
|
+
#
|
71
|
+
# w.start_if do |start|
|
72
|
+
# start.condition(:always)
|
73
|
+
# end
|
74
|
+
# end
|
75
|
+
end
|
data/test/helper.rb
CHANGED
data/test/test_condition.rb
CHANGED
@@ -2,7 +2,7 @@ require File.dirname(__FILE__) + '/helper'
|
|
2
2
|
|
3
3
|
class TestCondition < Test::Unit::TestCase
|
4
4
|
def test_generate_should_return_an_object_corresponding_to_the_given_type
|
5
|
-
assert_equal Conditions::
|
5
|
+
assert_equal Conditions::ProcessRunning, Condition.generate(:process_running).class
|
6
6
|
end
|
7
7
|
|
8
8
|
def test_generate_should_raise_on_invalid_type
|
data/test/test_god.rb
CHANGED
@@ -2,14 +2,15 @@ require File.dirname(__FILE__) + '/helper'
|
|
2
2
|
|
3
3
|
class TestGod < Test::Unit::TestCase
|
4
4
|
def test_should_create_new_meddle
|
5
|
-
Meddle.expects(:new).with(:port => 1).returns(mock(:monitor => true
|
5
|
+
Meddle.expects(:new).with(:port => 1).returns(mock(:monitor => true))
|
6
|
+
Timer.expects(:get).returns(stub(:join => nil)).times(2)
|
6
7
|
|
7
8
|
God.meddle(:port => 1) {}
|
8
9
|
end
|
9
10
|
|
10
11
|
def test_should_start_monitoring
|
11
12
|
Meddle.any_instance.expects(:monitor)
|
12
|
-
|
13
|
+
Timer.expects(:get).returns(stub(:join => nil)).times(2)
|
13
14
|
God.meddle {}
|
14
15
|
end
|
15
16
|
end
|
data/test/test_metric.rb
ADDED
@@ -0,0 +1,60 @@
|
|
1
|
+
require File.dirname(__FILE__) + '/helper'
|
2
|
+
|
3
|
+
class TestMetric < Test::Unit::TestCase
|
4
|
+
def setup
|
5
|
+
@metric = Metric.new(stub(:interval => 10), nil)
|
6
|
+
end
|
7
|
+
|
8
|
+
# watch
|
9
|
+
|
10
|
+
def test_watch
|
11
|
+
w = stub()
|
12
|
+
m = Metric.new(w, nil)
|
13
|
+
assert_equal w, m.watch
|
14
|
+
end
|
15
|
+
|
16
|
+
# destination
|
17
|
+
|
18
|
+
def test_destination
|
19
|
+
d = stub()
|
20
|
+
m = Metric.new(nil, d)
|
21
|
+
assert_equal d, m.destination
|
22
|
+
end
|
23
|
+
|
24
|
+
# condition
|
25
|
+
|
26
|
+
def test_condition_should_be_block_optional
|
27
|
+
@metric.condition(:fake_poll_condition)
|
28
|
+
assert_equal 1, @metric.conditions.size
|
29
|
+
end
|
30
|
+
|
31
|
+
def test_poll_condition_should_inherit_interval_from_watch_if_not_specified
|
32
|
+
@metric.condition(:fake_poll_condition)
|
33
|
+
assert_equal 10, @metric.conditions.first.interval
|
34
|
+
end
|
35
|
+
|
36
|
+
def test_poll_condition_should_abort_if_no_interval_and_no_watch_interval
|
37
|
+
metric = Metric.new(stub(:name => 'foo', :interval => nil), nil)
|
38
|
+
|
39
|
+
assert_raise AbortCalledError do
|
40
|
+
metric.condition(:fake_poll_condition)
|
41
|
+
end
|
42
|
+
end
|
43
|
+
|
44
|
+
def test_condition_should_allow_generation_of_subclasses_of_poll_or_event
|
45
|
+
metric = Metric.new(stub(:name => 'foo', :interval => 10), nil)
|
46
|
+
|
47
|
+
assert_nothing_raised do
|
48
|
+
metric.condition(:fake_poll_condition)
|
49
|
+
metric.condition(:fake_event_condition)
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
def test_condition_should_abort_if_not_subclass_of_poll_or_event
|
54
|
+
metric = Metric.new(stub(:name => 'foo', :interval => 10), nil)
|
55
|
+
|
56
|
+
assert_raise AbortCalledError do
|
57
|
+
metric.condition(:fake_condition) { |c| }
|
58
|
+
end
|
59
|
+
end
|
60
|
+
end
|
data/test/test_timer.rb
CHANGED
@@ -2,42 +2,52 @@ require File.dirname(__FILE__) + '/helper'
|
|
2
2
|
|
3
3
|
class TestTimer < Test::Unit::TestCase
|
4
4
|
def setup
|
5
|
-
|
5
|
+
Timer.reset
|
6
|
+
@t = Timer.get
|
6
7
|
end
|
7
8
|
|
8
9
|
def test_new_timer_should_have_no_events
|
9
10
|
assert_equal 0, @t.events.size
|
10
11
|
end
|
11
12
|
|
12
|
-
def
|
13
|
+
def test_schedule_should_queue_event
|
13
14
|
Time.stubs(:now).returns(0)
|
14
15
|
|
15
16
|
w = Watch.new(nil)
|
16
|
-
@t.
|
17
|
+
@t.schedule(stub(:interval => 20))
|
17
18
|
|
18
19
|
assert_equal 1, @t.events.size
|
19
|
-
assert_equal w, @t.events.first.watch
|
20
20
|
end
|
21
21
|
|
22
22
|
def test_timer_should_remove_expired_events
|
23
|
-
@t.
|
23
|
+
@t.schedule(stub(:interval => 0))
|
24
24
|
sleep(0.3)
|
25
25
|
assert_equal 0, @t.events.size
|
26
26
|
end
|
27
27
|
|
28
28
|
def test_timer_should_remove_only_expired_events
|
29
|
-
@t.
|
30
|
-
@t.
|
29
|
+
@t.schedule(stub(:interval => 0))
|
30
|
+
@t.schedule(stub(:interval => 1000))
|
31
31
|
sleep(0.3)
|
32
32
|
assert_equal 1, @t.events.size
|
33
33
|
end
|
34
34
|
|
35
35
|
def test_timer_should_sort_timer_events
|
36
|
-
@t.
|
37
|
-
@t.
|
38
|
-
@t.
|
39
|
-
@t.
|
36
|
+
@t.schedule(stub(:interval => 1000))
|
37
|
+
@t.schedule(stub(:interval => 800))
|
38
|
+
@t.schedule(stub(:interval => 900))
|
39
|
+
@t.schedule(stub(:interval => 100))
|
40
40
|
sleep(0.3)
|
41
41
|
assert_equal [100, 800, 900, 1000], @t.events.map { |x| x.condition.interval }
|
42
42
|
end
|
43
|
+
|
44
|
+
def test_unschedule_should_remove_conditions
|
45
|
+
a = stub()
|
46
|
+
b = stub()
|
47
|
+
@t.schedule(a, 100)
|
48
|
+
@t.schedule(b, 200)
|
49
|
+
assert_equal 2, @t.events.size
|
50
|
+
@t.unschedule(a)
|
51
|
+
assert_equal 1, @t.events.size
|
52
|
+
end
|
43
53
|
end
|