eye 0.2.1 → 0.2.2
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +24 -17
- data/bin/eye +5 -0
- data/examples/notify.eye +18 -0
- data/examples/process_thin.rb +29 -0
- data/examples/sidekiq.eye +5 -1
- data/examples/test.eye +24 -17
- data/examples/thin-farm.eye +29 -0
- data/examples/unicorn.eye +2 -0
- data/eye.gemspec +1 -1
- data/lib/eye.rb +1 -1
- data/lib/eye/checker.rb +3 -5
- data/lib/eye/checker/http.rb +1 -2
- data/lib/eye/checker/socket.rb +2 -2
- data/lib/eye/controller/commands.rb +2 -0
- data/lib/eye/controller/send_command.rb +8 -2
- data/lib/eye/dsl.rb +1 -0
- data/lib/eye/dsl/opts.rb +36 -2
- data/lib/eye/dsl/pure_opts.rb +17 -39
- data/lib/eye/{checker → dsl}/validation.rb +15 -5
- data/lib/eye/group.rb +13 -4
- data/lib/eye/group/chain.rb +11 -0
- data/lib/eye/notify.rb +1 -1
- data/lib/eye/notify/mail.rb +1 -1
- data/lib/eye/process/child.rb +0 -2
- data/lib/eye/process/commands.rb +11 -14
- data/lib/eye/process/monitor.rb +10 -16
- data/lib/eye/process/scheduler.rb +8 -3
- data/lib/eye/process/states.rb +1 -2
- data/lib/eye/process/system.rb +38 -7
- data/lib/eye/process/watchers.rb +3 -3
- data/lib/eye/settings.rb +1 -1
- data/lib/eye/system.rb +33 -16
- data/lib/eye/trigger.rb +1 -1
- data/lib/eye/utils/celluloid_chain.rb +2 -0
- data/spec/checker/cpu_spec.rb +1 -1
- data/spec/checker/http_spec.rb +2 -2
- data/spec/checker/memory_spec.rb +2 -2
- data/spec/checker_spec.rb +1 -1
- data/spec/controller/controller_spec.rb +6 -0
- data/spec/controller/find_objects_spec.rb +6 -0
- data/spec/controller/intergration_spec.rb +24 -0
- data/spec/dsl/checks_spec.rb +2 -2
- data/spec/dsl/notify_spec.rb +12 -3
- data/spec/dsl/with_server_spec.rb +26 -2
- data/spec/process/checks/child_checks_spec.rb +1 -1
- data/spec/process/checks/memory_spec.rb +14 -0
- data/spec/process/scheduler_spec.rb +8 -0
- data/spec/process/system_spec.rb +27 -4
- data/spec/spec_helper.rb +3 -1
- data/spec/system_spec.rb +12 -5
- data/spec/utils/celluloid_chain_spec.rb +8 -0
- metadata +8 -5
data/lib/eye/dsl/pure_opts.rb
CHANGED
@@ -66,49 +66,10 @@ class Eye::Dsl::PureOpts
|
|
66
66
|
[]
|
67
67
|
end
|
68
68
|
|
69
|
-
# execute part of config on particular server
|
70
|
-
# array of strings
|
71
|
-
# regexp
|
72
|
-
# string
|
73
|
-
def with_server(glob = nil, &block)
|
74
|
-
on_server = true
|
75
|
-
|
76
|
-
if glob.present?
|
77
|
-
host = Eye::System.host
|
78
|
-
|
79
|
-
if glob.is_a?(Array)
|
80
|
-
on_server = !!glob.any?{|elem| elem == host}
|
81
|
-
elsif glob.is_a?(Regexp)
|
82
|
-
on_server = !!host.match(glob)
|
83
|
-
elsif glob.is_a?(String) || glob.is_a?(Symbol)
|
84
|
-
on_server = (host == glob.to_s)
|
85
|
-
end
|
86
|
-
end
|
87
|
-
|
88
|
-
with_condition(on_server, &block)
|
89
|
-
|
90
|
-
on_server
|
91
|
-
end
|
92
|
-
|
93
69
|
def with_condition(cond = true, &block)
|
94
70
|
self.instance_eval(&block) if cond && block
|
95
71
|
end
|
96
72
|
|
97
|
-
def self.with_parsed_file(file_name)
|
98
|
-
saved_parsed_filename = Eye.parsed_filename
|
99
|
-
|
100
|
-
require 'pathname'
|
101
|
-
|
102
|
-
real_filename = Eye.parsed_filename && File.symlink?(Eye.parsed_filename) ? File.readlink(Eye.parsed_filename) : Eye.parsed_filename
|
103
|
-
dirname = File.dirname(real_filename) rescue nil
|
104
|
-
path = Pathname.new(file_name).expand_path(dirname).to_s
|
105
|
-
|
106
|
-
Eye.parsed_filename = path
|
107
|
-
yield path
|
108
|
-
ensure
|
109
|
-
Eye.parsed_filename = saved_parsed_filename
|
110
|
-
end
|
111
|
-
|
112
73
|
def include(proc, *args)
|
113
74
|
if proc.is_a?(String)
|
114
75
|
self.class.with_parsed_file(proc) do |path|
|
@@ -129,4 +90,21 @@ class Eye::Dsl::PureOpts
|
|
129
90
|
end
|
130
91
|
end
|
131
92
|
|
93
|
+
private
|
94
|
+
|
95
|
+
def self.with_parsed_file(file_name)
|
96
|
+
saved_parsed_filename = Eye.parsed_filename
|
97
|
+
|
98
|
+
require 'pathname'
|
99
|
+
|
100
|
+
real_filename = Eye.parsed_filename && File.symlink?(Eye.parsed_filename) ? File.readlink(Eye.parsed_filename) : Eye.parsed_filename
|
101
|
+
dirname = File.dirname(real_filename) rescue nil
|
102
|
+
path = Pathname.new(file_name).expand_path(dirname).to_s
|
103
|
+
|
104
|
+
Eye.parsed_filename = path
|
105
|
+
yield path
|
106
|
+
ensure
|
107
|
+
Eye.parsed_filename = saved_parsed_filename
|
108
|
+
end
|
109
|
+
|
132
110
|
end
|
@@ -1,24 +1,27 @@
|
|
1
|
-
module Eye::
|
1
|
+
module Eye::Dsl::Validation
|
2
2
|
class Error < Exception; end
|
3
3
|
|
4
4
|
def inherited(subclass)
|
5
5
|
subclass.validates = self.validates.clone
|
6
6
|
subclass.should_bes = self.should_bes.clone
|
7
7
|
subclass.defaults = self.defaults.clone
|
8
|
+
subclass.variants = self.variants.clone
|
8
9
|
end
|
9
10
|
|
10
|
-
attr_accessor :validates, :should_bes, :defaults
|
11
|
+
attr_accessor :validates, :should_bes, :defaults, :variants
|
11
12
|
|
12
13
|
def validates; @validates ||= {}; end
|
13
14
|
def should_bes; @should_bes ||= []; end
|
14
15
|
def defaults; @defaults ||= {}; end
|
16
|
+
def variants; @variants ||= {}; end
|
15
17
|
|
16
|
-
def param(param, types = [], should_be = false, default = nil)
|
18
|
+
def param(param, types = [], should_be = false, default = nil, _variants = nil)
|
17
19
|
param = param.to_sym
|
18
20
|
|
19
21
|
validates[param] = types
|
20
22
|
should_bes << param if should_be
|
21
23
|
defaults[param] = default
|
24
|
+
variants[param] = _variants
|
22
25
|
|
23
26
|
define_method "#{param}" do
|
24
27
|
@options[param.to_sym] || default
|
@@ -27,13 +30,20 @@ module Eye::Checker::Validation
|
|
27
30
|
|
28
31
|
def validate(options = {})
|
29
32
|
options.each do |param, value|
|
30
|
-
|
33
|
+
param = param.to_sym
|
34
|
+
types = validates[param]
|
31
35
|
unless types
|
32
|
-
if param
|
36
|
+
if param != :type
|
33
37
|
raise Error, "#{self.name} unknown param :#{param} value #{value.inspect}"
|
34
38
|
end
|
35
39
|
end
|
36
40
|
|
41
|
+
if self.variants[param]
|
42
|
+
if value && !value.is_a?(Proc) && !self.variants[param].include?(value)
|
43
|
+
raise Error, "#{value.inspect} should within #{self.variants[param].inspect}"
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
37
47
|
next if types.blank?
|
38
48
|
|
39
49
|
types = Array(types)
|
data/lib/eye/group.rb
CHANGED
@@ -68,10 +68,13 @@ class Eye::Group
|
|
68
68
|
def send_command(command, *args)
|
69
69
|
info "send_command: #{command}"
|
70
70
|
|
71
|
-
|
72
|
-
delete
|
73
|
-
|
74
|
-
|
71
|
+
case command
|
72
|
+
when :delete
|
73
|
+
delete *args
|
74
|
+
when :break_chain
|
75
|
+
break_chain *args
|
76
|
+
else
|
77
|
+
schedule command, *args, "#{command} by user"
|
75
78
|
end
|
76
79
|
end
|
77
80
|
|
@@ -104,6 +107,12 @@ class Eye::Group
|
|
104
107
|
async_schedule :signal, sig
|
105
108
|
end
|
106
109
|
|
110
|
+
def break_chain
|
111
|
+
info "break chain"
|
112
|
+
scheduler.clear_pending_list
|
113
|
+
@chain_breaker = true
|
114
|
+
end
|
115
|
+
|
107
116
|
def clear
|
108
117
|
@processes = Eye::Utils::AliveArray.new
|
109
118
|
end
|
data/lib/eye/group/chain.rb
CHANGED
@@ -7,6 +7,9 @@ private
|
|
7
7
|
|
8
8
|
@chain_processes_count = @processes.size
|
9
9
|
@chain_processes_current = 0
|
10
|
+
@chain_breaker = false
|
11
|
+
|
12
|
+
started_at = Time.now
|
10
13
|
|
11
14
|
@processes.each do | process |
|
12
15
|
chain_schedule_process(process, type, command, *args)
|
@@ -15,11 +18,16 @@ private
|
|
15
18
|
|
16
19
|
# to skip last sleep
|
17
20
|
break if @chain_processes_current.to_i == @chain_processes_count.to_i
|
21
|
+
break if @chain_breaker
|
18
22
|
|
19
23
|
# wait next process
|
20
24
|
sleep grace.to_f
|
25
|
+
|
26
|
+
break if @chain_breaker
|
21
27
|
end
|
22
28
|
|
29
|
+
debug "chain finished #{Time.now - started_at}s"
|
30
|
+
|
23
31
|
@chain_processes_count = nil
|
24
32
|
@chain_processes_current = nil
|
25
33
|
end
|
@@ -29,6 +37,9 @@ private
|
|
29
37
|
|
30
38
|
if type == :sync
|
31
39
|
# sync command, with waiting
|
40
|
+
# this is very hackety, because call method of the process without its scheduler
|
41
|
+
# need to provide some scheduler future
|
42
|
+
process.last_scheduled_reason = self.last_scheduled_reason
|
32
43
|
process.send(command, *args)
|
33
44
|
else
|
34
45
|
# async command
|
data/lib/eye/notify.rb
CHANGED
data/lib/eye/notify/mail.rb
CHANGED
@@ -13,7 +13,7 @@ class Eye::Notify::Mail < Eye::Notify
|
|
13
13
|
param :domain, String
|
14
14
|
param :user, String
|
15
15
|
param :password, String
|
16
|
-
param :auth, Symbol
|
16
|
+
param :auth, Symbol, nil, nil, [:plain, :login, :cram_md5]
|
17
17
|
|
18
18
|
param :from_mail, String
|
19
19
|
param :from_name, String, nil, 'eye'
|
data/lib/eye/process/child.rb
CHANGED
data/lib/eye/process/commands.rb
CHANGED
@@ -17,8 +17,8 @@ module Eye::Process::Commands
|
|
17
17
|
debug "process (#{self.pid}) ok started"
|
18
18
|
switch :started
|
19
19
|
else
|
20
|
-
|
21
|
-
if
|
20
|
+
error "process (#{self.pid}) failed to start (#{result[:error].inspect})"
|
21
|
+
if process_realy_running?
|
22
22
|
warn "kill, what remains from process (#{self.pid}), because its failed to start (without pid_file impossible to monitoring)"
|
23
23
|
send_signal(:KILL)
|
24
24
|
sleep 0.2 # little grace
|
@@ -53,7 +53,7 @@ module Eye::Process::Commands
|
|
53
53
|
switch :stopped
|
54
54
|
|
55
55
|
if control_pid?
|
56
|
-
info "delete pid_file: #{self[:
|
56
|
+
info "delete pid_file: #{self[:pid_file_ex]}"
|
57
57
|
clear_pid_file
|
58
58
|
end
|
59
59
|
|
@@ -135,7 +135,7 @@ private
|
|
135
135
|
|
136
136
|
sleep self[:stop_grace].to_f
|
137
137
|
|
138
|
-
# if process not die here, by default we kill it
|
138
|
+
# if process not die here, by default we force kill it
|
139
139
|
if process_realy_running?
|
140
140
|
warn "process not die after TERM and stop_grace #{self[:stop_grace].to_f}s, so send KILL"
|
141
141
|
send_signal(:KILL)
|
@@ -188,17 +188,14 @@ private
|
|
188
188
|
sleep self[:start_grace].to_f
|
189
189
|
|
190
190
|
unless process_realy_running?
|
191
|
-
error "process with pid
|
191
|
+
error "process with pid(#{self.pid}) not found, may be crashed (#{check_logs_str})"
|
192
192
|
return {:error => :not_realy_running}
|
193
193
|
end
|
194
194
|
|
195
|
-
|
196
|
-
|
197
|
-
rescue => ex
|
198
|
-
error "save pid to file raised with #{ex.inspect}"
|
199
|
-
return {:error => :cant_write_pid}
|
195
|
+
unless failsafe_save_pid
|
196
|
+
return {:error => :cant_write_pid}
|
200
197
|
end
|
201
|
-
|
198
|
+
|
202
199
|
res
|
203
200
|
end
|
204
201
|
|
@@ -226,17 +223,17 @@ private
|
|
226
223
|
sleep self[:start_grace].to_f
|
227
224
|
|
228
225
|
unless set_pid_from_file
|
229
|
-
error "pid_file(#{self[:
|
226
|
+
error "pid_file(#{self[:pid_file_ex]}) does not appears after start_grace #{self[:start_grace].to_f}, check start_command, or tune start_grace (eye dont know what to monitor without pid)"
|
230
227
|
return {:error => :pid_not_found}
|
231
228
|
end
|
232
229
|
|
233
230
|
unless process_realy_running?
|
234
|
-
error "process in pid_file(#{self[:
|
231
|
+
error "process in pid_file(#{self[:pid_file_ex]})(#{self.pid}) not found, maybe process do not write there actual pid, or just crashed (#{check_logs_str})"
|
235
232
|
return {:error => :not_realy_running}
|
236
233
|
end
|
237
234
|
|
238
235
|
res[:pid] = self.pid
|
239
|
-
info "process get pid:#{res[:pid]}"
|
236
|
+
info "process get pid:#{res[:pid]}, pid_file #{self[:pid_file_ex]}"
|
240
237
|
res
|
241
238
|
end
|
242
239
|
|
data/lib/eye/process/monitor.rb
CHANGED
@@ -38,7 +38,7 @@ private
|
|
38
38
|
|
39
39
|
# check that process runned
|
40
40
|
unless process_realy_running?
|
41
|
-
warn
|
41
|
+
warn "check_alive: process(#{self.pid}) not found!"
|
42
42
|
notify :warn, 'crashed!'
|
43
43
|
switch :crashed
|
44
44
|
else
|
@@ -49,14 +49,20 @@ private
|
|
49
49
|
msg = "check_alive: pid_file(#{self[:pid_file]}) changes by itself (pid:#{self.pid}) => (pid:#{ppid})"
|
50
50
|
if control_pid?
|
51
51
|
msg += ", not correct, pid_file is under eye control, so rewrited back pid:#{self.pid}"
|
52
|
-
|
52
|
+
unless failsafe_save_pid
|
53
|
+
msg += ', (Can`t rewrite pid_file O_o)'
|
54
|
+
end
|
53
55
|
else
|
54
56
|
if ppid == nil
|
55
57
|
msg += ', rewrited because empty'
|
56
|
-
|
58
|
+
unless failsafe_save_pid
|
59
|
+
msg += ', (Can`t rewrite pid_file O_o)'
|
60
|
+
end
|
57
61
|
elsif (Time.now - pid_file_ctime > REWRITE_FACKUP_PIDFILE_PERIOD)
|
58
62
|
msg += ", > #{REWRITE_FACKUP_PIDFILE_PERIOD.inspect} ago, so rewrited (even if pid_file not under eye control)"
|
59
|
-
|
63
|
+
unless failsafe_save_pid
|
64
|
+
msg += ', (Can`t rewrite pid_file O_o)'
|
65
|
+
end
|
60
66
|
else
|
61
67
|
msg += ', not under eye control, so ignored'
|
62
68
|
end
|
@@ -68,18 +74,6 @@ private
|
|
68
74
|
end
|
69
75
|
end
|
70
76
|
|
71
|
-
def failsafe_load_pid
|
72
|
-
pid = load_pid_from_file
|
73
|
-
|
74
|
-
if !pid
|
75
|
-
# this is can be symlink changed case
|
76
|
-
sleep 0.1
|
77
|
-
pid = load_pid_from_file
|
78
|
-
end
|
79
|
-
|
80
|
-
pid
|
81
|
-
end
|
82
|
-
|
83
77
|
def check_crash
|
84
78
|
if down?
|
85
79
|
if self[:keep_alive] && !@flapping
|
@@ -3,18 +3,23 @@ module Eye::Process::Scheduler
|
|
3
3
|
# ex: schedule :update_config, config, "reason: update_config"
|
4
4
|
def schedule(command, *args, &block)
|
5
5
|
if scheduler.alive?
|
6
|
+
unless self.respond_to?(command)
|
7
|
+
warn "object not support :#{command} to schedule"
|
8
|
+
return
|
9
|
+
end
|
10
|
+
|
6
11
|
reason = if args.present? && [String, Symbol].include?(args[-1].class)
|
7
12
|
args.pop
|
8
13
|
end
|
9
14
|
|
10
|
-
info "schedule :#{command} (#{reason})"
|
15
|
+
info "schedule :#{command} #{reason ? "(reason: #{reason})" : nil}"
|
11
16
|
scheduler.add_wo_dups(:scheduled_action, command, {:args => args, :reason => reason}, &block)
|
12
17
|
end
|
13
18
|
end
|
14
19
|
|
15
20
|
def scheduled_action(command, h = {}, &block)
|
16
21
|
reason = h.delete(:reason)
|
17
|
-
info "=> #{command} #{h[:args].present? ? "#{h[:args]*',' }" : nil}(#{reason})"
|
22
|
+
info "=> #{command} #{h[:args].present? ? "#{h[:args]*',' }" : nil} #{reason ? "(reason: #{reason})" : nil}"
|
18
23
|
|
19
24
|
@current_scheduled_command = command
|
20
25
|
@last_scheduled_command = command
|
@@ -47,4 +52,4 @@ private
|
|
47
52
|
@scheduler ||= Eye::Utils::CelluloidChain.new(current_actor)
|
48
53
|
end
|
49
54
|
|
50
|
-
end
|
55
|
+
end
|
data/lib/eye/process/states.rb
CHANGED
@@ -1,4 +1,3 @@
|
|
1
|
-
gem 'state_machine'
|
2
1
|
require 'state_machine'
|
3
2
|
|
4
3
|
class Eye::Process
|
@@ -82,7 +81,7 @@ class Eye::Process
|
|
82
81
|
|
83
82
|
def log_transition(transition)
|
84
83
|
@states_history.push transition.to_name, @state_reason
|
85
|
-
info "switch :#{transition.event} [:#{transition.from_name} => :#{transition.to_name}] #{@state_reason ? "(#{@state_reason})" : nil}"
|
84
|
+
info "switch :#{transition.event} [:#{transition.from_name} => :#{transition.to_name}] #{@state_reason ? "(reason: #{@state_reason})" : nil}"
|
86
85
|
end
|
87
86
|
|
88
87
|
def upd_for_triggers(transition)
|
data/lib/eye/process/system.rb
CHANGED
@@ -18,9 +18,10 @@ module Eye::Process::System
|
|
18
18
|
File.open(self[:pid_file_ex], 'w') do |f|
|
19
19
|
f.write self.pid
|
20
20
|
end
|
21
|
+
true
|
22
|
+
else
|
23
|
+
false
|
21
24
|
end
|
22
|
-
|
23
|
-
true
|
24
25
|
end
|
25
26
|
|
26
27
|
def clear_pid_file
|
@@ -35,16 +36,26 @@ module Eye::Process::System
|
|
35
36
|
end
|
36
37
|
|
37
38
|
def process_realy_running?
|
38
|
-
|
39
|
+
if self.pid
|
40
|
+
res = Eye::System.check_pid_alive(self.pid)
|
41
|
+
if res[:error] && res[:error].class != Errno::ESRCH
|
42
|
+
error "process_realy_running?: check_pid_alive returns '#{res[:error].message}'"
|
43
|
+
end
|
44
|
+
res[:result]
|
45
|
+
else
|
46
|
+
debug "process_realy_running?: called without pid"
|
47
|
+
nil
|
48
|
+
end
|
39
49
|
end
|
40
50
|
|
41
51
|
def send_signal(code)
|
42
|
-
debug "send signal #{code}"
|
43
|
-
|
44
52
|
res = Eye::System.send_signal(self.pid, code)
|
45
|
-
error(res[:message]) if res[:status] != :ok
|
46
53
|
|
47
|
-
|
54
|
+
msg = "send_signal #{code} to #{self.pid}"
|
55
|
+
msg += ", error<#{res[:error]}>" if res[:error]
|
56
|
+
info msg
|
57
|
+
|
58
|
+
res[:result] == :ok
|
48
59
|
end
|
49
60
|
|
50
61
|
# non blocking actor timeout
|
@@ -56,6 +67,26 @@ module Eye::Process::System
|
|
56
67
|
defer{ Eye::System::execute cmd, cfg }
|
57
68
|
end
|
58
69
|
|
70
|
+
def failsafe_load_pid
|
71
|
+
pid = load_pid_from_file
|
72
|
+
|
73
|
+
if !pid
|
74
|
+
# this is can be symlink changed case
|
75
|
+
sleep 0.1
|
76
|
+
pid = load_pid_from_file
|
77
|
+
end
|
78
|
+
|
79
|
+
pid
|
80
|
+
end
|
81
|
+
|
82
|
+
def failsafe_save_pid
|
83
|
+
save_pid_to_file
|
84
|
+
true
|
85
|
+
rescue => ex
|
86
|
+
error "failsafe_save_pid: #{ex.message}"
|
87
|
+
false
|
88
|
+
end
|
89
|
+
|
59
90
|
private
|
60
91
|
|
61
92
|
def wait_for_condition_sync(timeout, step, &block)
|