god 0.4.3 → 0.5.0
Sign up to get free protection for your applications and to get access to all the features.
- data/History.txt +43 -7
- data/Manifest.txt +20 -4
- data/Rakefile +1 -1
- data/bin/god +263 -195
- data/examples/events.god +66 -34
- data/examples/gravatar.god +25 -12
- data/init/god +42 -0
- data/lib/god/behavior.rb +9 -29
- data/lib/god/behaviors/clean_pid_file.rb +6 -2
- data/lib/god/behaviors/notify_when_flapping.rb +4 -4
- data/lib/god/condition.rb +48 -6
- data/lib/god/conditions/always.rb +5 -1
- data/lib/god/conditions/cpu_usage.rb +13 -5
- data/lib/god/conditions/degrading_lambda.rb +8 -3
- data/lib/god/conditions/flapping.rb +97 -0
- data/lib/god/conditions/http_response_code.rb +97 -0
- data/lib/god/conditions/lambda.rb +8 -2
- data/lib/god/conditions/memory_usage.rb +13 -5
- data/lib/god/conditions/process_exits.rb +11 -3
- data/lib/god/conditions/process_running.rb +22 -4
- data/lib/god/conditions/tries.rb +16 -5
- data/lib/god/configurable.rb +54 -0
- data/lib/god/contact.rb +106 -0
- data/lib/god/contacts/email.rb +73 -0
- data/lib/god/errors.rb +3 -0
- data/lib/god/hub.rb +138 -33
- data/lib/god/logger.rb +21 -4
- data/lib/god/metric.rb +3 -4
- data/lib/god/process.rb +93 -49
- data/lib/god/socket.rb +60 -0
- data/lib/god/task.rb +233 -0
- data/lib/god/trigger.rb +43 -0
- data/lib/god/watch.rb +48 -114
- data/lib/god.rb +216 -63
- data/test/configs/child_events/child_events.god +20 -1
- data/test/configs/child_polls/child_polls.god +26 -6
- data/test/configs/child_polls/simple_server.rb +10 -1
- data/test/configs/contact/contact.god +74 -0
- data/test/configs/contact/simple_server.rb +3 -0
- data/test/configs/daemon_events/daemon_events.god +5 -2
- data/test/configs/daemon_events/simple_server.rb +2 -0
- data/test/configs/daemon_events/simple_server_stop.rb +9 -0
- data/test/configs/degrading_lambda/degrading_lambda.god +1 -3
- data/test/configs/task/logs/.placeholder +0 -0
- data/test/configs/task/task.god +26 -0
- data/test/helper.rb +19 -11
- data/test/test_conditions_http_response_code.rb +115 -0
- data/test/test_conditions_process_running.rb +2 -2
- data/test/test_conditions_tries.rb +21 -0
- data/test/test_contact.rb +109 -0
- data/test/test_god.rb +101 -17
- data/test/test_hub.rb +64 -1
- data/test/test_process.rb +43 -56
- data/test/{test_server.rb → test_socket.rb} +6 -20
- data/test/test_task.rb +86 -0
- data/test/test_trigger.rb +59 -0
- data/test/test_watch.rb +32 -7
- metadata +27 -8
- data/lib/god/reporter.rb +0 -25
- data/lib/god/server.rb +0 -37
- data/test/test_reporter.rb +0 -18
data/lib/god/task.rb
ADDED
@@ -0,0 +1,233 @@
|
|
1
|
+
module God
|
2
|
+
|
3
|
+
class Task
|
4
|
+
attr_accessor :name, :interval, :group, :valid_states, :initial_state
|
5
|
+
|
6
|
+
attr_writer :autostart
|
7
|
+
def autostart?; @autostart; end
|
8
|
+
|
9
|
+
# api
|
10
|
+
attr_accessor :state, :behaviors, :metrics
|
11
|
+
|
12
|
+
# internal
|
13
|
+
attr_accessor :mutex
|
14
|
+
|
15
|
+
def initialize
|
16
|
+
@autostart ||= true
|
17
|
+
|
18
|
+
# initial state is unmonitored
|
19
|
+
self.state = :unmonitored
|
20
|
+
|
21
|
+
# the list of behaviors
|
22
|
+
self.behaviors = []
|
23
|
+
|
24
|
+
# the list of conditions for each action
|
25
|
+
self.metrics = {nil => [], :unmonitored => []}
|
26
|
+
|
27
|
+
# mutex
|
28
|
+
self.mutex = Monitor.new
|
29
|
+
end
|
30
|
+
|
31
|
+
def prepare
|
32
|
+
self.valid_states.each do |state|
|
33
|
+
self.metrics[state] ||= []
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def valid?
|
38
|
+
valid = true
|
39
|
+
|
40
|
+
# a name must be specified
|
41
|
+
if self.name.nil?
|
42
|
+
valid = false
|
43
|
+
LOG.log(self, :error, "No name was specified")
|
44
|
+
end
|
45
|
+
|
46
|
+
# valid_states must be specified
|
47
|
+
if self.valid_states.nil?
|
48
|
+
valid = false
|
49
|
+
LOG.log(self, :error, "No valid_states array was specified")
|
50
|
+
end
|
51
|
+
|
52
|
+
# valid_states must be specified
|
53
|
+
if self.initial_state.nil?
|
54
|
+
valid = false
|
55
|
+
LOG.log(self, :error, "No initial_state was specified")
|
56
|
+
end
|
57
|
+
|
58
|
+
valid
|
59
|
+
end
|
60
|
+
|
61
|
+
###########################################################################
|
62
|
+
#
|
63
|
+
# Advanced mode
|
64
|
+
#
|
65
|
+
###########################################################################
|
66
|
+
|
67
|
+
def canonical_hash_form(to)
|
68
|
+
to.instance_of?(Symbol) ? {true => to} : to
|
69
|
+
end
|
70
|
+
|
71
|
+
# Define a transition handler which consists of a set of conditions
|
72
|
+
def transition(start_states, end_states)
|
73
|
+
# convert end_states into canonical hash form
|
74
|
+
canonical_end_states = canonical_hash_form(end_states)
|
75
|
+
|
76
|
+
Array(start_states).each do |start_state|
|
77
|
+
# validate start state
|
78
|
+
unless self.valid_states.include?(start_state)
|
79
|
+
abort "Invalid state :#{start_state}. Must be one of the symbols #{self.valid_states.map{|x| ":#{x}"}.join(', ')}"
|
80
|
+
end
|
81
|
+
|
82
|
+
# create a new metric to hold the watch, end states, and conditions
|
83
|
+
m = Metric.new(self, canonical_end_states)
|
84
|
+
|
85
|
+
if block_given?
|
86
|
+
# let the config file define some conditions on the metric
|
87
|
+
yield(m)
|
88
|
+
else
|
89
|
+
# add an :always condition if no block
|
90
|
+
m.condition(:always) do |c|
|
91
|
+
c.what = true
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
# record the metric
|
96
|
+
self.metrics[start_state] ||= []
|
97
|
+
self.metrics[start_state] << m
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
def lifecycle
|
102
|
+
# create a new metric to hold the watch and conditions
|
103
|
+
m = Metric.new(self)
|
104
|
+
|
105
|
+
# let the config file define some conditions on the metric
|
106
|
+
yield(m)
|
107
|
+
|
108
|
+
# record the metric
|
109
|
+
self.metrics[nil] << m
|
110
|
+
end
|
111
|
+
|
112
|
+
###########################################################################
|
113
|
+
#
|
114
|
+
# Lifecycle
|
115
|
+
#
|
116
|
+
###########################################################################
|
117
|
+
|
118
|
+
# Enable monitoring
|
119
|
+
def monitor
|
120
|
+
self.move(self.initial_state)
|
121
|
+
end
|
122
|
+
|
123
|
+
# Disable monitoring
|
124
|
+
def unmonitor
|
125
|
+
self.move(:unmonitored)
|
126
|
+
end
|
127
|
+
|
128
|
+
# Move from one state to another
|
129
|
+
def move(to_state)
|
130
|
+
self.mutex.synchronize do
|
131
|
+
orig_to_state = to_state
|
132
|
+
from_state = self.state
|
133
|
+
|
134
|
+
msg = "#{self.name} move '#{from_state}' to '#{to_state}'"
|
135
|
+
Syslog.debug(msg)
|
136
|
+
LOG.log(self, :info, msg)
|
137
|
+
|
138
|
+
# cleanup from current state
|
139
|
+
self.metrics[from_state].each { |m| m.disable }
|
140
|
+
|
141
|
+
if to_state == :unmonitored
|
142
|
+
self.metrics[nil].each { |m| m.disable }
|
143
|
+
end
|
144
|
+
|
145
|
+
# perform action
|
146
|
+
self.action(to_state)
|
147
|
+
|
148
|
+
# enable simple mode
|
149
|
+
if [:start, :restart].include?(to_state) && self.metrics[to_state].empty?
|
150
|
+
to_state = :up
|
151
|
+
end
|
152
|
+
|
153
|
+
# move to new state
|
154
|
+
self.metrics[to_state].each { |m| m.enable }
|
155
|
+
|
156
|
+
# if no from state, enable lifecycle metric
|
157
|
+
if from_state == :unmonitored
|
158
|
+
self.metrics[nil].each { |m| m.enable }
|
159
|
+
end
|
160
|
+
|
161
|
+
# set state
|
162
|
+
self.state = to_state
|
163
|
+
|
164
|
+
# trigger
|
165
|
+
Trigger.broadcast(self, :state_change, [from_state, orig_to_state])
|
166
|
+
|
167
|
+
# return self
|
168
|
+
self
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
###########################################################################
|
173
|
+
#
|
174
|
+
# Actions
|
175
|
+
#
|
176
|
+
###########################################################################
|
177
|
+
|
178
|
+
def method_missing(sym, *args)
|
179
|
+
unless (sym.to_s =~ /=$/)
|
180
|
+
super
|
181
|
+
end
|
182
|
+
|
183
|
+
base = sym.to_s.chop.intern
|
184
|
+
|
185
|
+
unless self.valid_states.include?(base)
|
186
|
+
super
|
187
|
+
end
|
188
|
+
|
189
|
+
self.class.send(:attr_accessor, base)
|
190
|
+
self.send(sym, *args)
|
191
|
+
end
|
192
|
+
|
193
|
+
# +a+ is the action Symbol
|
194
|
+
# +c+ is the Condition
|
195
|
+
def action(a, c = nil)
|
196
|
+
if self.respond_to?(a)
|
197
|
+
command = self.send(a)
|
198
|
+
|
199
|
+
case command
|
200
|
+
when String
|
201
|
+
msg = "#{self.name} #{a}: #{command}"
|
202
|
+
Syslog.debug(msg)
|
203
|
+
LOG.log(self, :info, msg)
|
204
|
+
|
205
|
+
system(command)
|
206
|
+
when Proc
|
207
|
+
msg = "#{self.name} #{a}: lambda"
|
208
|
+
Syslog.debug(msg)
|
209
|
+
LOG.log(self, :info, msg)
|
210
|
+
|
211
|
+
command.call
|
212
|
+
else
|
213
|
+
raise NotImplementedError
|
214
|
+
end
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
218
|
+
###########################################################################
|
219
|
+
#
|
220
|
+
# Registration
|
221
|
+
#
|
222
|
+
###########################################################################
|
223
|
+
|
224
|
+
def register!
|
225
|
+
# override if necessary
|
226
|
+
end
|
227
|
+
|
228
|
+
def unregister!
|
229
|
+
# override if necessary
|
230
|
+
end
|
231
|
+
end
|
232
|
+
|
233
|
+
end
|
data/lib/god/trigger.rb
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
module God
|
2
|
+
|
3
|
+
class Trigger
|
4
|
+
|
5
|
+
class << self
|
6
|
+
attr_accessor :triggers # {task.name => condition}
|
7
|
+
end
|
8
|
+
|
9
|
+
# init
|
10
|
+
self.triggers = {}
|
11
|
+
@mutex = Mutex.new
|
12
|
+
|
13
|
+
def self.register(condition)
|
14
|
+
@mutex.synchronize do
|
15
|
+
self.triggers[condition.watch.name] ||= []
|
16
|
+
self.triggers[condition.watch.name] << condition
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.deregister(condition)
|
21
|
+
@mutex.synchronize do
|
22
|
+
self.triggers[condition.watch.name].delete(condition)
|
23
|
+
self.triggers.delete(condition.watch.name) if self.triggers[condition.watch.name].empty?
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.broadcast(task, message, payload)
|
28
|
+
return unless self.triggers[task.name]
|
29
|
+
|
30
|
+
@mutex.synchronize do
|
31
|
+
self.triggers[task.name].each do |t|
|
32
|
+
t.process(message, payload)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.reset
|
38
|
+
self.triggers.clear
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
|
43
|
+
end
|
data/lib/god/watch.rb
CHANGED
@@ -3,51 +3,33 @@ require 'forwardable'
|
|
3
3
|
|
4
4
|
module God
|
5
5
|
|
6
|
-
class Watch
|
6
|
+
class Watch < Task
|
7
7
|
VALID_STATES = [:init, :up, :start, :restart]
|
8
|
+
INITIAL_STATE = :init
|
8
9
|
|
9
10
|
# config
|
10
|
-
attr_accessor :
|
11
|
-
:grace, :start_grace, :stop_grace, :restart_grace
|
12
|
-
|
13
|
-
|
14
|
-
attr_writer :autostart
|
15
|
-
def autostart?; @autostart; end
|
11
|
+
attr_accessor :grace, :start_grace, :stop_grace, :restart_grace
|
16
12
|
|
17
13
|
extend Forwardable
|
18
14
|
def_delegators :@process, :name, :uid, :gid, :start, :stop, :restart,
|
19
15
|
:name=, :uid=, :gid=, :start=, :stop=, :restart=,
|
20
|
-
:pid_file, :pid_file=, :log, :log=, :alive?
|
21
|
-
|
22
|
-
# api
|
23
|
-
attr_accessor :behaviors, :metrics
|
24
|
-
|
25
|
-
# internal
|
26
|
-
attr_accessor :mutex
|
27
|
-
|
16
|
+
:pid_file, :pid_file=, :log, :log=, :alive?
|
28
17
|
#
|
29
18
|
def initialize
|
30
|
-
|
19
|
+
super
|
20
|
+
|
31
21
|
@process = God::Process.new
|
32
|
-
|
33
|
-
# no grace period by default
|
34
|
-
self.grace = self.start_grace = self.stop_grace = self.restart_grace = 0
|
35
22
|
|
36
|
-
#
|
37
|
-
self.
|
23
|
+
# valid states
|
24
|
+
self.valid_states = VALID_STATES
|
25
|
+
self.initial_state = INITIAL_STATE
|
38
26
|
|
39
|
-
#
|
40
|
-
self.
|
41
|
-
:start => [],
|
42
|
-
:restart => [],
|
43
|
-
:up => []}
|
44
|
-
|
45
|
-
# mutex
|
46
|
-
self.mutex = Mutex.new
|
27
|
+
# no grace period by default
|
28
|
+
self.grace = self.start_grace = self.stop_grace = self.restart_grace = 0
|
47
29
|
end
|
48
30
|
|
49
31
|
def valid?
|
50
|
-
@process.valid?
|
32
|
+
super && @process.valid?
|
51
33
|
end
|
52
34
|
|
53
35
|
###########################################################################
|
@@ -74,35 +56,6 @@ module God
|
|
74
56
|
self.behaviors << b
|
75
57
|
end
|
76
58
|
|
77
|
-
###########################################################################
|
78
|
-
#
|
79
|
-
# Advanced mode
|
80
|
-
#
|
81
|
-
###########################################################################
|
82
|
-
|
83
|
-
# Define a transition handler which consists of a set of conditions
|
84
|
-
def transition(start_states, end_states)
|
85
|
-
# convert to into canonical hash form
|
86
|
-
canonical_end_states = canonical_hash_form(end_states)
|
87
|
-
|
88
|
-
# for each start state do
|
89
|
-
Array(start_states).each do |start_state|
|
90
|
-
# validate start state
|
91
|
-
unless VALID_STATES.include?(start_state)
|
92
|
-
abort "Invalid state :#{start_state}. Must be one of the symbols #{VALID_STATES.map{|x| ":#{x}"}.join(', ')}"
|
93
|
-
end
|
94
|
-
|
95
|
-
# create a new metric to hold the watch, end states, and conditions
|
96
|
-
m = Metric.new(self, canonical_end_states)
|
97
|
-
|
98
|
-
# let the config file define some conditions on the metric
|
99
|
-
yield(m)
|
100
|
-
|
101
|
-
# record the metric
|
102
|
-
self.metrics[start_state] << m
|
103
|
-
end
|
104
|
-
end
|
105
|
-
|
106
59
|
###########################################################################
|
107
60
|
#
|
108
61
|
# Simple mode
|
@@ -126,7 +79,7 @@ module God
|
|
126
79
|
# Lifecycle
|
127
80
|
#
|
128
81
|
###########################################################################
|
129
|
-
|
82
|
+
|
130
83
|
# Enable monitoring
|
131
84
|
def monitor
|
132
85
|
# start monitoring at the first available of the init or up states
|
@@ -137,56 +90,19 @@ module God
|
|
137
90
|
end
|
138
91
|
end
|
139
92
|
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
# Move from one state to another
|
146
|
-
def move(to_state)
|
147
|
-
msg = "#{self.name} move '#{self.state}' to '#{to_state}'"
|
148
|
-
Syslog.debug(msg)
|
149
|
-
LOG.log(self, :info, msg)
|
150
|
-
|
151
|
-
# cleanup from current state
|
152
|
-
from_state = self.state
|
153
|
-
if from_state
|
154
|
-
self.metrics[from_state].each { |m| m.disable }
|
155
|
-
end
|
156
|
-
|
157
|
-
# perform action
|
158
|
-
self.action(to_state)
|
159
|
-
|
160
|
-
# enable simple mode
|
161
|
-
if [:start, :restart].include?(to_state) && self.metrics[to_state].empty?
|
162
|
-
to_state = :up
|
163
|
-
end
|
164
|
-
|
165
|
-
# move to new state
|
166
|
-
if to_state
|
167
|
-
self.metrics[to_state].each { |m| m.enable }
|
168
|
-
end
|
169
|
-
|
170
|
-
# set state
|
171
|
-
self.state = to_state
|
172
|
-
|
173
|
-
# return self
|
174
|
-
self
|
175
|
-
end
|
93
|
+
###########################################################################
|
94
|
+
#
|
95
|
+
# Actions
|
96
|
+
#
|
97
|
+
###########################################################################
|
176
98
|
|
177
99
|
def action(a, c = nil)
|
178
100
|
case a
|
179
101
|
when :start
|
180
|
-
msg = "#{self.name} start: #{self.start.to_s}"
|
181
|
-
Syslog.debug(msg)
|
182
|
-
LOG.log(self, :info, msg)
|
183
102
|
call_action(c, :start)
|
184
103
|
sleep(self.start_grace + self.grace)
|
185
104
|
when :restart
|
186
105
|
if self.restart
|
187
|
-
msg = "#{self.name} restart: #{self.restart.to_s}"
|
188
|
-
Syslog.debug(msg)
|
189
|
-
LOG.log(self, :info, msg)
|
190
106
|
call_action(c, :restart)
|
191
107
|
else
|
192
108
|
action(:stop, c)
|
@@ -194,33 +110,51 @@ module God
|
|
194
110
|
end
|
195
111
|
sleep(self.restart_grace + self.grace)
|
196
112
|
when :stop
|
197
|
-
if self.stop
|
198
|
-
msg = "#{self.name} stop: #{self.stop.to_s}"
|
199
|
-
Syslog.debug(msg)
|
200
|
-
LOG.log(self, :info, msg)
|
201
|
-
end
|
202
113
|
call_action(c, :stop)
|
203
114
|
sleep(self.stop_grace + self.grace)
|
204
|
-
end
|
115
|
+
end
|
205
116
|
end
|
206
117
|
|
207
118
|
def call_action(condition, action)
|
208
119
|
# before
|
209
120
|
before_items = self.behaviors
|
210
121
|
before_items += [condition] if condition
|
211
|
-
before_items.each
|
122
|
+
before_items.each do |b|
|
123
|
+
info = b.send("before_#{action}")
|
124
|
+
if info
|
125
|
+
msg = "#{self.name} before_#{action}: #{info} (#{b.base_name})"
|
126
|
+
Syslog.debug(msg)
|
127
|
+
LOG.log(self, :info, msg)
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
# log
|
132
|
+
if self.send(action)
|
133
|
+
msg = "#{self.name} #{action}: #{self.send(action).to_s}"
|
134
|
+
Syslog.debug(msg)
|
135
|
+
LOG.log(self, :info, msg)
|
136
|
+
end
|
212
137
|
|
213
138
|
@process.call_action(action)
|
214
|
-
|
139
|
+
|
215
140
|
# after
|
216
141
|
after_items = self.behaviors
|
217
142
|
after_items += [condition] if condition
|
218
|
-
after_items.each
|
143
|
+
after_items.each do |b|
|
144
|
+
info = b.send("after_#{action}")
|
145
|
+
if info
|
146
|
+
msg = "#{self.name} after_#{action}: #{info} (#{b.base_name})"
|
147
|
+
Syslog.debug(msg)
|
148
|
+
LOG.log(self, :info, msg)
|
149
|
+
end
|
150
|
+
end
|
219
151
|
end
|
220
152
|
|
221
|
-
|
222
|
-
|
223
|
-
|
153
|
+
###########################################################################
|
154
|
+
#
|
155
|
+
# Registration
|
156
|
+
#
|
157
|
+
###########################################################################
|
224
158
|
|
225
159
|
def register!
|
226
160
|
God.registry.add(@process)
|