god 0.4.3 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/History.txt +43 -7
- data/Manifest.txt +20 -4
- data/Rakefile +1 -1
- data/bin/god +263 -195
- data/examples/events.god +66 -34
- data/examples/gravatar.god +25 -12
- data/init/god +42 -0
- data/lib/god/behavior.rb +9 -29
- data/lib/god/behaviors/clean_pid_file.rb +6 -2
- data/lib/god/behaviors/notify_when_flapping.rb +4 -4
- data/lib/god/condition.rb +48 -6
- data/lib/god/conditions/always.rb +5 -1
- data/lib/god/conditions/cpu_usage.rb +13 -5
- data/lib/god/conditions/degrading_lambda.rb +8 -3
- data/lib/god/conditions/flapping.rb +97 -0
- data/lib/god/conditions/http_response_code.rb +97 -0
- data/lib/god/conditions/lambda.rb +8 -2
- data/lib/god/conditions/memory_usage.rb +13 -5
- data/lib/god/conditions/process_exits.rb +11 -3
- data/lib/god/conditions/process_running.rb +22 -4
- data/lib/god/conditions/tries.rb +16 -5
- data/lib/god/configurable.rb +54 -0
- data/lib/god/contact.rb +106 -0
- data/lib/god/contacts/email.rb +73 -0
- data/lib/god/errors.rb +3 -0
- data/lib/god/hub.rb +138 -33
- data/lib/god/logger.rb +21 -4
- data/lib/god/metric.rb +3 -4
- data/lib/god/process.rb +93 -49
- data/lib/god/socket.rb +60 -0
- data/lib/god/task.rb +233 -0
- data/lib/god/trigger.rb +43 -0
- data/lib/god/watch.rb +48 -114
- data/lib/god.rb +216 -63
- data/test/configs/child_events/child_events.god +20 -1
- data/test/configs/child_polls/child_polls.god +26 -6
- data/test/configs/child_polls/simple_server.rb +10 -1
- data/test/configs/contact/contact.god +74 -0
- data/test/configs/contact/simple_server.rb +3 -0
- data/test/configs/daemon_events/daemon_events.god +5 -2
- data/test/configs/daemon_events/simple_server.rb +2 -0
- data/test/configs/daemon_events/simple_server_stop.rb +9 -0
- data/test/configs/degrading_lambda/degrading_lambda.god +1 -3
- data/test/configs/task/logs/.placeholder +0 -0
- data/test/configs/task/task.god +26 -0
- data/test/helper.rb +19 -11
- data/test/test_conditions_http_response_code.rb +115 -0
- data/test/test_conditions_process_running.rb +2 -2
- data/test/test_conditions_tries.rb +21 -0
- data/test/test_contact.rb +109 -0
- data/test/test_god.rb +101 -17
- data/test/test_hub.rb +64 -1
- data/test/test_process.rb +43 -56
- data/test/{test_server.rb → test_socket.rb} +6 -20
- data/test/test_task.rb +86 -0
- data/test/test_trigger.rb +59 -0
- data/test/test_watch.rb +32 -7
- metadata +27 -8
- data/lib/god/reporter.rb +0 -25
- data/lib/god/server.rb +0 -37
- data/test/test_reporter.rb +0 -18
data/lib/god/task.rb
ADDED
@@ -0,0 +1,233 @@
|
|
1
|
+
module God
|
2
|
+
|
3
|
+
class Task
|
4
|
+
attr_accessor :name, :interval, :group, :valid_states, :initial_state
|
5
|
+
|
6
|
+
attr_writer :autostart
|
7
|
+
def autostart?; @autostart; end
|
8
|
+
|
9
|
+
# api
|
10
|
+
attr_accessor :state, :behaviors, :metrics
|
11
|
+
|
12
|
+
# internal
|
13
|
+
attr_accessor :mutex
|
14
|
+
|
15
|
+
def initialize
|
16
|
+
@autostart ||= true
|
17
|
+
|
18
|
+
# initial state is unmonitored
|
19
|
+
self.state = :unmonitored
|
20
|
+
|
21
|
+
# the list of behaviors
|
22
|
+
self.behaviors = []
|
23
|
+
|
24
|
+
# the list of conditions for each action
|
25
|
+
self.metrics = {nil => [], :unmonitored => []}
|
26
|
+
|
27
|
+
# mutex
|
28
|
+
self.mutex = Monitor.new
|
29
|
+
end
|
30
|
+
|
31
|
+
def prepare
|
32
|
+
self.valid_states.each do |state|
|
33
|
+
self.metrics[state] ||= []
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def valid?
|
38
|
+
valid = true
|
39
|
+
|
40
|
+
# a name must be specified
|
41
|
+
if self.name.nil?
|
42
|
+
valid = false
|
43
|
+
LOG.log(self, :error, "No name was specified")
|
44
|
+
end
|
45
|
+
|
46
|
+
# valid_states must be specified
|
47
|
+
if self.valid_states.nil?
|
48
|
+
valid = false
|
49
|
+
LOG.log(self, :error, "No valid_states array was specified")
|
50
|
+
end
|
51
|
+
|
52
|
+
# valid_states must be specified
|
53
|
+
if self.initial_state.nil?
|
54
|
+
valid = false
|
55
|
+
LOG.log(self, :error, "No initial_state was specified")
|
56
|
+
end
|
57
|
+
|
58
|
+
valid
|
59
|
+
end
|
60
|
+
|
61
|
+
###########################################################################
|
62
|
+
#
|
63
|
+
# Advanced mode
|
64
|
+
#
|
65
|
+
###########################################################################
|
66
|
+
|
67
|
+
def canonical_hash_form(to)
|
68
|
+
to.instance_of?(Symbol) ? {true => to} : to
|
69
|
+
end
|
70
|
+
|
71
|
+
# Define a transition handler which consists of a set of conditions
|
72
|
+
def transition(start_states, end_states)
|
73
|
+
# convert end_states into canonical hash form
|
74
|
+
canonical_end_states = canonical_hash_form(end_states)
|
75
|
+
|
76
|
+
Array(start_states).each do |start_state|
|
77
|
+
# validate start state
|
78
|
+
unless self.valid_states.include?(start_state)
|
79
|
+
abort "Invalid state :#{start_state}. Must be one of the symbols #{self.valid_states.map{|x| ":#{x}"}.join(', ')}"
|
80
|
+
end
|
81
|
+
|
82
|
+
# create a new metric to hold the watch, end states, and conditions
|
83
|
+
m = Metric.new(self, canonical_end_states)
|
84
|
+
|
85
|
+
if block_given?
|
86
|
+
# let the config file define some conditions on the metric
|
87
|
+
yield(m)
|
88
|
+
else
|
89
|
+
# add an :always condition if no block
|
90
|
+
m.condition(:always) do |c|
|
91
|
+
c.what = true
|
92
|
+
end
|
93
|
+
end
|
94
|
+
|
95
|
+
# record the metric
|
96
|
+
self.metrics[start_state] ||= []
|
97
|
+
self.metrics[start_state] << m
|
98
|
+
end
|
99
|
+
end
|
100
|
+
|
101
|
+
def lifecycle
|
102
|
+
# create a new metric to hold the watch and conditions
|
103
|
+
m = Metric.new(self)
|
104
|
+
|
105
|
+
# let the config file define some conditions on the metric
|
106
|
+
yield(m)
|
107
|
+
|
108
|
+
# record the metric
|
109
|
+
self.metrics[nil] << m
|
110
|
+
end
|
111
|
+
|
112
|
+
###########################################################################
|
113
|
+
#
|
114
|
+
# Lifecycle
|
115
|
+
#
|
116
|
+
###########################################################################
|
117
|
+
|
118
|
+
# Enable monitoring
|
119
|
+
def monitor
|
120
|
+
self.move(self.initial_state)
|
121
|
+
end
|
122
|
+
|
123
|
+
# Disable monitoring
|
124
|
+
def unmonitor
|
125
|
+
self.move(:unmonitored)
|
126
|
+
end
|
127
|
+
|
128
|
+
# Move from one state to another
|
129
|
+
def move(to_state)
|
130
|
+
self.mutex.synchronize do
|
131
|
+
orig_to_state = to_state
|
132
|
+
from_state = self.state
|
133
|
+
|
134
|
+
msg = "#{self.name} move '#{from_state}' to '#{to_state}'"
|
135
|
+
Syslog.debug(msg)
|
136
|
+
LOG.log(self, :info, msg)
|
137
|
+
|
138
|
+
# cleanup from current state
|
139
|
+
self.metrics[from_state].each { |m| m.disable }
|
140
|
+
|
141
|
+
if to_state == :unmonitored
|
142
|
+
self.metrics[nil].each { |m| m.disable }
|
143
|
+
end
|
144
|
+
|
145
|
+
# perform action
|
146
|
+
self.action(to_state)
|
147
|
+
|
148
|
+
# enable simple mode
|
149
|
+
if [:start, :restart].include?(to_state) && self.metrics[to_state].empty?
|
150
|
+
to_state = :up
|
151
|
+
end
|
152
|
+
|
153
|
+
# move to new state
|
154
|
+
self.metrics[to_state].each { |m| m.enable }
|
155
|
+
|
156
|
+
# if no from state, enable lifecycle metric
|
157
|
+
if from_state == :unmonitored
|
158
|
+
self.metrics[nil].each { |m| m.enable }
|
159
|
+
end
|
160
|
+
|
161
|
+
# set state
|
162
|
+
self.state = to_state
|
163
|
+
|
164
|
+
# trigger
|
165
|
+
Trigger.broadcast(self, :state_change, [from_state, orig_to_state])
|
166
|
+
|
167
|
+
# return self
|
168
|
+
self
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
###########################################################################
|
173
|
+
#
|
174
|
+
# Actions
|
175
|
+
#
|
176
|
+
###########################################################################
|
177
|
+
|
178
|
+
def method_missing(sym, *args)
|
179
|
+
unless (sym.to_s =~ /=$/)
|
180
|
+
super
|
181
|
+
end
|
182
|
+
|
183
|
+
base = sym.to_s.chop.intern
|
184
|
+
|
185
|
+
unless self.valid_states.include?(base)
|
186
|
+
super
|
187
|
+
end
|
188
|
+
|
189
|
+
self.class.send(:attr_accessor, base)
|
190
|
+
self.send(sym, *args)
|
191
|
+
end
|
192
|
+
|
193
|
+
# +a+ is the action Symbol
|
194
|
+
# +c+ is the Condition
|
195
|
+
def action(a, c = nil)
|
196
|
+
if self.respond_to?(a)
|
197
|
+
command = self.send(a)
|
198
|
+
|
199
|
+
case command
|
200
|
+
when String
|
201
|
+
msg = "#{self.name} #{a}: #{command}"
|
202
|
+
Syslog.debug(msg)
|
203
|
+
LOG.log(self, :info, msg)
|
204
|
+
|
205
|
+
system(command)
|
206
|
+
when Proc
|
207
|
+
msg = "#{self.name} #{a}: lambda"
|
208
|
+
Syslog.debug(msg)
|
209
|
+
LOG.log(self, :info, msg)
|
210
|
+
|
211
|
+
command.call
|
212
|
+
else
|
213
|
+
raise NotImplementedError
|
214
|
+
end
|
215
|
+
end
|
216
|
+
end
|
217
|
+
|
218
|
+
###########################################################################
|
219
|
+
#
|
220
|
+
# Registration
|
221
|
+
#
|
222
|
+
###########################################################################
|
223
|
+
|
224
|
+
def register!
|
225
|
+
# override if necessary
|
226
|
+
end
|
227
|
+
|
228
|
+
def unregister!
|
229
|
+
# override if necessary
|
230
|
+
end
|
231
|
+
end
|
232
|
+
|
233
|
+
end
|
data/lib/god/trigger.rb
ADDED
@@ -0,0 +1,43 @@
|
|
1
|
+
module God
|
2
|
+
|
3
|
+
class Trigger
|
4
|
+
|
5
|
+
class << self
|
6
|
+
attr_accessor :triggers # {task.name => condition}
|
7
|
+
end
|
8
|
+
|
9
|
+
# init
|
10
|
+
self.triggers = {}
|
11
|
+
@mutex = Mutex.new
|
12
|
+
|
13
|
+
def self.register(condition)
|
14
|
+
@mutex.synchronize do
|
15
|
+
self.triggers[condition.watch.name] ||= []
|
16
|
+
self.triggers[condition.watch.name] << condition
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def self.deregister(condition)
|
21
|
+
@mutex.synchronize do
|
22
|
+
self.triggers[condition.watch.name].delete(condition)
|
23
|
+
self.triggers.delete(condition.watch.name) if self.triggers[condition.watch.name].empty?
|
24
|
+
end
|
25
|
+
end
|
26
|
+
|
27
|
+
def self.broadcast(task, message, payload)
|
28
|
+
return unless self.triggers[task.name]
|
29
|
+
|
30
|
+
@mutex.synchronize do
|
31
|
+
self.triggers[task.name].each do |t|
|
32
|
+
t.process(message, payload)
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|
36
|
+
|
37
|
+
def self.reset
|
38
|
+
self.triggers.clear
|
39
|
+
end
|
40
|
+
|
41
|
+
end
|
42
|
+
|
43
|
+
end
|
data/lib/god/watch.rb
CHANGED
@@ -3,51 +3,33 @@ require 'forwardable'
|
|
3
3
|
|
4
4
|
module God
|
5
5
|
|
6
|
-
class Watch
|
6
|
+
class Watch < Task
|
7
7
|
VALID_STATES = [:init, :up, :start, :restart]
|
8
|
+
INITIAL_STATE = :init
|
8
9
|
|
9
10
|
# config
|
10
|
-
attr_accessor :
|
11
|
-
:grace, :start_grace, :stop_grace, :restart_grace
|
12
|
-
|
13
|
-
|
14
|
-
attr_writer :autostart
|
15
|
-
def autostart?; @autostart; end
|
11
|
+
attr_accessor :grace, :start_grace, :stop_grace, :restart_grace
|
16
12
|
|
17
13
|
extend Forwardable
|
18
14
|
def_delegators :@process, :name, :uid, :gid, :start, :stop, :restart,
|
19
15
|
:name=, :uid=, :gid=, :start=, :stop=, :restart=,
|
20
|
-
:pid_file, :pid_file=, :log, :log=, :alive?
|
21
|
-
|
22
|
-
# api
|
23
|
-
attr_accessor :behaviors, :metrics
|
24
|
-
|
25
|
-
# internal
|
26
|
-
attr_accessor :mutex
|
27
|
-
|
16
|
+
:pid_file, :pid_file=, :log, :log=, :alive?
|
28
17
|
#
|
29
18
|
def initialize
|
30
|
-
|
19
|
+
super
|
20
|
+
|
31
21
|
@process = God::Process.new
|
32
|
-
|
33
|
-
# no grace period by default
|
34
|
-
self.grace = self.start_grace = self.stop_grace = self.restart_grace = 0
|
35
22
|
|
36
|
-
#
|
37
|
-
self.
|
23
|
+
# valid states
|
24
|
+
self.valid_states = VALID_STATES
|
25
|
+
self.initial_state = INITIAL_STATE
|
38
26
|
|
39
|
-
#
|
40
|
-
self.
|
41
|
-
:start => [],
|
42
|
-
:restart => [],
|
43
|
-
:up => []}
|
44
|
-
|
45
|
-
# mutex
|
46
|
-
self.mutex = Mutex.new
|
27
|
+
# no grace period by default
|
28
|
+
self.grace = self.start_grace = self.stop_grace = self.restart_grace = 0
|
47
29
|
end
|
48
30
|
|
49
31
|
def valid?
|
50
|
-
@process.valid?
|
32
|
+
super && @process.valid?
|
51
33
|
end
|
52
34
|
|
53
35
|
###########################################################################
|
@@ -74,35 +56,6 @@ module God
|
|
74
56
|
self.behaviors << b
|
75
57
|
end
|
76
58
|
|
77
|
-
###########################################################################
|
78
|
-
#
|
79
|
-
# Advanced mode
|
80
|
-
#
|
81
|
-
###########################################################################
|
82
|
-
|
83
|
-
# Define a transition handler which consists of a set of conditions
|
84
|
-
def transition(start_states, end_states)
|
85
|
-
# convert to into canonical hash form
|
86
|
-
canonical_end_states = canonical_hash_form(end_states)
|
87
|
-
|
88
|
-
# for each start state do
|
89
|
-
Array(start_states).each do |start_state|
|
90
|
-
# validate start state
|
91
|
-
unless VALID_STATES.include?(start_state)
|
92
|
-
abort "Invalid state :#{start_state}. Must be one of the symbols #{VALID_STATES.map{|x| ":#{x}"}.join(', ')}"
|
93
|
-
end
|
94
|
-
|
95
|
-
# create a new metric to hold the watch, end states, and conditions
|
96
|
-
m = Metric.new(self, canonical_end_states)
|
97
|
-
|
98
|
-
# let the config file define some conditions on the metric
|
99
|
-
yield(m)
|
100
|
-
|
101
|
-
# record the metric
|
102
|
-
self.metrics[start_state] << m
|
103
|
-
end
|
104
|
-
end
|
105
|
-
|
106
59
|
###########################################################################
|
107
60
|
#
|
108
61
|
# Simple mode
|
@@ -126,7 +79,7 @@ module God
|
|
126
79
|
# Lifecycle
|
127
80
|
#
|
128
81
|
###########################################################################
|
129
|
-
|
82
|
+
|
130
83
|
# Enable monitoring
|
131
84
|
def monitor
|
132
85
|
# start monitoring at the first available of the init or up states
|
@@ -137,56 +90,19 @@ module God
|
|
137
90
|
end
|
138
91
|
end
|
139
92
|
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
145
|
-
# Move from one state to another
|
146
|
-
def move(to_state)
|
147
|
-
msg = "#{self.name} move '#{self.state}' to '#{to_state}'"
|
148
|
-
Syslog.debug(msg)
|
149
|
-
LOG.log(self, :info, msg)
|
150
|
-
|
151
|
-
# cleanup from current state
|
152
|
-
from_state = self.state
|
153
|
-
if from_state
|
154
|
-
self.metrics[from_state].each { |m| m.disable }
|
155
|
-
end
|
156
|
-
|
157
|
-
# perform action
|
158
|
-
self.action(to_state)
|
159
|
-
|
160
|
-
# enable simple mode
|
161
|
-
if [:start, :restart].include?(to_state) && self.metrics[to_state].empty?
|
162
|
-
to_state = :up
|
163
|
-
end
|
164
|
-
|
165
|
-
# move to new state
|
166
|
-
if to_state
|
167
|
-
self.metrics[to_state].each { |m| m.enable }
|
168
|
-
end
|
169
|
-
|
170
|
-
# set state
|
171
|
-
self.state = to_state
|
172
|
-
|
173
|
-
# return self
|
174
|
-
self
|
175
|
-
end
|
93
|
+
###########################################################################
|
94
|
+
#
|
95
|
+
# Actions
|
96
|
+
#
|
97
|
+
###########################################################################
|
176
98
|
|
177
99
|
def action(a, c = nil)
|
178
100
|
case a
|
179
101
|
when :start
|
180
|
-
msg = "#{self.name} start: #{self.start.to_s}"
|
181
|
-
Syslog.debug(msg)
|
182
|
-
LOG.log(self, :info, msg)
|
183
102
|
call_action(c, :start)
|
184
103
|
sleep(self.start_grace + self.grace)
|
185
104
|
when :restart
|
186
105
|
if self.restart
|
187
|
-
msg = "#{self.name} restart: #{self.restart.to_s}"
|
188
|
-
Syslog.debug(msg)
|
189
|
-
LOG.log(self, :info, msg)
|
190
106
|
call_action(c, :restart)
|
191
107
|
else
|
192
108
|
action(:stop, c)
|
@@ -194,33 +110,51 @@ module God
|
|
194
110
|
end
|
195
111
|
sleep(self.restart_grace + self.grace)
|
196
112
|
when :stop
|
197
|
-
if self.stop
|
198
|
-
msg = "#{self.name} stop: #{self.stop.to_s}"
|
199
|
-
Syslog.debug(msg)
|
200
|
-
LOG.log(self, :info, msg)
|
201
|
-
end
|
202
113
|
call_action(c, :stop)
|
203
114
|
sleep(self.stop_grace + self.grace)
|
204
|
-
end
|
115
|
+
end
|
205
116
|
end
|
206
117
|
|
207
118
|
def call_action(condition, action)
|
208
119
|
# before
|
209
120
|
before_items = self.behaviors
|
210
121
|
before_items += [condition] if condition
|
211
|
-
before_items.each
|
122
|
+
before_items.each do |b|
|
123
|
+
info = b.send("before_#{action}")
|
124
|
+
if info
|
125
|
+
msg = "#{self.name} before_#{action}: #{info} (#{b.base_name})"
|
126
|
+
Syslog.debug(msg)
|
127
|
+
LOG.log(self, :info, msg)
|
128
|
+
end
|
129
|
+
end
|
130
|
+
|
131
|
+
# log
|
132
|
+
if self.send(action)
|
133
|
+
msg = "#{self.name} #{action}: #{self.send(action).to_s}"
|
134
|
+
Syslog.debug(msg)
|
135
|
+
LOG.log(self, :info, msg)
|
136
|
+
end
|
212
137
|
|
213
138
|
@process.call_action(action)
|
214
|
-
|
139
|
+
|
215
140
|
# after
|
216
141
|
after_items = self.behaviors
|
217
142
|
after_items += [condition] if condition
|
218
|
-
after_items.each
|
143
|
+
after_items.each do |b|
|
144
|
+
info = b.send("after_#{action}")
|
145
|
+
if info
|
146
|
+
msg = "#{self.name} after_#{action}: #{info} (#{b.base_name})"
|
147
|
+
Syslog.debug(msg)
|
148
|
+
LOG.log(self, :info, msg)
|
149
|
+
end
|
150
|
+
end
|
219
151
|
end
|
220
152
|
|
221
|
-
|
222
|
-
|
223
|
-
|
153
|
+
###########################################################################
|
154
|
+
#
|
155
|
+
# Registration
|
156
|
+
#
|
157
|
+
###########################################################################
|
224
158
|
|
225
159
|
def register!
|
226
160
|
God.registry.add(@process)
|