eye 0.2.1 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +24 -17
- data/bin/eye +5 -0
- data/examples/notify.eye +18 -0
- data/examples/process_thin.rb +29 -0
- data/examples/sidekiq.eye +5 -1
- data/examples/test.eye +24 -17
- data/examples/thin-farm.eye +29 -0
- data/examples/unicorn.eye +2 -0
- data/eye.gemspec +1 -1
- data/lib/eye.rb +1 -1
- data/lib/eye/checker.rb +3 -5
- data/lib/eye/checker/http.rb +1 -2
- data/lib/eye/checker/socket.rb +2 -2
- data/lib/eye/controller/commands.rb +2 -0
- data/lib/eye/controller/send_command.rb +8 -2
- data/lib/eye/dsl.rb +1 -0
- data/lib/eye/dsl/opts.rb +36 -2
- data/lib/eye/dsl/pure_opts.rb +17 -39
- data/lib/eye/{checker → dsl}/validation.rb +15 -5
- data/lib/eye/group.rb +13 -4
- data/lib/eye/group/chain.rb +11 -0
- data/lib/eye/notify.rb +1 -1
- data/lib/eye/notify/mail.rb +1 -1
- data/lib/eye/process/child.rb +0 -2
- data/lib/eye/process/commands.rb +11 -14
- data/lib/eye/process/monitor.rb +10 -16
- data/lib/eye/process/scheduler.rb +8 -3
- data/lib/eye/process/states.rb +1 -2
- data/lib/eye/process/system.rb +38 -7
- data/lib/eye/process/watchers.rb +3 -3
- data/lib/eye/settings.rb +1 -1
- data/lib/eye/system.rb +33 -16
- data/lib/eye/trigger.rb +1 -1
- data/lib/eye/utils/celluloid_chain.rb +2 -0
- data/spec/checker/cpu_spec.rb +1 -1
- data/spec/checker/http_spec.rb +2 -2
- data/spec/checker/memory_spec.rb +2 -2
- data/spec/checker_spec.rb +1 -1
- data/spec/controller/controller_spec.rb +6 -0
- data/spec/controller/find_objects_spec.rb +6 -0
- data/spec/controller/intergration_spec.rb +24 -0
- data/spec/dsl/checks_spec.rb +2 -2
- data/spec/dsl/notify_spec.rb +12 -3
- data/spec/dsl/with_server_spec.rb +26 -2
- data/spec/process/checks/child_checks_spec.rb +1 -1
- data/spec/process/checks/memory_spec.rb +14 -0
- data/spec/process/scheduler_spec.rb +8 -0
- data/spec/process/system_spec.rb +27 -4
- data/spec/spec_helper.rb +3 -1
- data/spec/system_spec.rb +12 -5
- data/spec/utils/celluloid_chain_spec.rb +8 -0
- metadata +8 -5
data/lib/eye/dsl/pure_opts.rb
CHANGED
@@ -66,49 +66,10 @@ class Eye::Dsl::PureOpts
|
|
66
66
|
[]
|
67
67
|
end
|
68
68
|
|
69
|
-
# execute part of config on particular server
|
70
|
-
# array of strings
|
71
|
-
# regexp
|
72
|
-
# string
|
73
|
-
def with_server(glob = nil, &block)
|
74
|
-
on_server = true
|
75
|
-
|
76
|
-
if glob.present?
|
77
|
-
host = Eye::System.host
|
78
|
-
|
79
|
-
if glob.is_a?(Array)
|
80
|
-
on_server = !!glob.any?{|elem| elem == host}
|
81
|
-
elsif glob.is_a?(Regexp)
|
82
|
-
on_server = !!host.match(glob)
|
83
|
-
elsif glob.is_a?(String) || glob.is_a?(Symbol)
|
84
|
-
on_server = (host == glob.to_s)
|
85
|
-
end
|
86
|
-
end
|
87
|
-
|
88
|
-
with_condition(on_server, &block)
|
89
|
-
|
90
|
-
on_server
|
91
|
-
end
|
92
|
-
|
93
69
|
def with_condition(cond = true, &block)
|
94
70
|
self.instance_eval(&block) if cond && block
|
95
71
|
end
|
96
72
|
|
97
|
-
def self.with_parsed_file(file_name)
|
98
|
-
saved_parsed_filename = Eye.parsed_filename
|
99
|
-
|
100
|
-
require 'pathname'
|
101
|
-
|
102
|
-
real_filename = Eye.parsed_filename && File.symlink?(Eye.parsed_filename) ? File.readlink(Eye.parsed_filename) : Eye.parsed_filename
|
103
|
-
dirname = File.dirname(real_filename) rescue nil
|
104
|
-
path = Pathname.new(file_name).expand_path(dirname).to_s
|
105
|
-
|
106
|
-
Eye.parsed_filename = path
|
107
|
-
yield path
|
108
|
-
ensure
|
109
|
-
Eye.parsed_filename = saved_parsed_filename
|
110
|
-
end
|
111
|
-
|
112
73
|
def include(proc, *args)
|
113
74
|
if proc.is_a?(String)
|
114
75
|
self.class.with_parsed_file(proc) do |path|
|
@@ -129,4 +90,21 @@ class Eye::Dsl::PureOpts
|
|
129
90
|
end
|
130
91
|
end
|
131
92
|
|
93
|
+
private
|
94
|
+
|
95
|
+
def self.with_parsed_file(file_name)
|
96
|
+
saved_parsed_filename = Eye.parsed_filename
|
97
|
+
|
98
|
+
require 'pathname'
|
99
|
+
|
100
|
+
real_filename = Eye.parsed_filename && File.symlink?(Eye.parsed_filename) ? File.readlink(Eye.parsed_filename) : Eye.parsed_filename
|
101
|
+
dirname = File.dirname(real_filename) rescue nil
|
102
|
+
path = Pathname.new(file_name).expand_path(dirname).to_s
|
103
|
+
|
104
|
+
Eye.parsed_filename = path
|
105
|
+
yield path
|
106
|
+
ensure
|
107
|
+
Eye.parsed_filename = saved_parsed_filename
|
108
|
+
end
|
109
|
+
|
132
110
|
end
|
@@ -1,24 +1,27 @@
|
|
1
|
-
module Eye::
|
1
|
+
module Eye::Dsl::Validation
|
2
2
|
class Error < Exception; end
|
3
3
|
|
4
4
|
def inherited(subclass)
|
5
5
|
subclass.validates = self.validates.clone
|
6
6
|
subclass.should_bes = self.should_bes.clone
|
7
7
|
subclass.defaults = self.defaults.clone
|
8
|
+
subclass.variants = self.variants.clone
|
8
9
|
end
|
9
10
|
|
10
|
-
attr_accessor :validates, :should_bes, :defaults
|
11
|
+
attr_accessor :validates, :should_bes, :defaults, :variants
|
11
12
|
|
12
13
|
def validates; @validates ||= {}; end
|
13
14
|
def should_bes; @should_bes ||= []; end
|
14
15
|
def defaults; @defaults ||= {}; end
|
16
|
+
def variants; @variants ||= {}; end
|
15
17
|
|
16
|
-
def param(param, types = [], should_be = false, default = nil)
|
18
|
+
def param(param, types = [], should_be = false, default = nil, _variants = nil)
|
17
19
|
param = param.to_sym
|
18
20
|
|
19
21
|
validates[param] = types
|
20
22
|
should_bes << param if should_be
|
21
23
|
defaults[param] = default
|
24
|
+
variants[param] = _variants
|
22
25
|
|
23
26
|
define_method "#{param}" do
|
24
27
|
@options[param.to_sym] || default
|
@@ -27,13 +30,20 @@ module Eye::Checker::Validation
|
|
27
30
|
|
28
31
|
def validate(options = {})
|
29
32
|
options.each do |param, value|
|
30
|
-
|
33
|
+
param = param.to_sym
|
34
|
+
types = validates[param]
|
31
35
|
unless types
|
32
|
-
if param
|
36
|
+
if param != :type
|
33
37
|
raise Error, "#{self.name} unknown param :#{param} value #{value.inspect}"
|
34
38
|
end
|
35
39
|
end
|
36
40
|
|
41
|
+
if self.variants[param]
|
42
|
+
if value && !value.is_a?(Proc) && !self.variants[param].include?(value)
|
43
|
+
raise Error, "#{value.inspect} should within #{self.variants[param].inspect}"
|
44
|
+
end
|
45
|
+
end
|
46
|
+
|
37
47
|
next if types.blank?
|
38
48
|
|
39
49
|
types = Array(types)
|
data/lib/eye/group.rb
CHANGED
@@ -68,10 +68,13 @@ class Eye::Group
|
|
68
68
|
def send_command(command, *args)
|
69
69
|
info "send_command: #{command}"
|
70
70
|
|
71
|
-
|
72
|
-
delete
|
73
|
-
|
74
|
-
|
71
|
+
case command
|
72
|
+
when :delete
|
73
|
+
delete *args
|
74
|
+
when :break_chain
|
75
|
+
break_chain *args
|
76
|
+
else
|
77
|
+
schedule command, *args, "#{command} by user"
|
75
78
|
end
|
76
79
|
end
|
77
80
|
|
@@ -104,6 +107,12 @@ class Eye::Group
|
|
104
107
|
async_schedule :signal, sig
|
105
108
|
end
|
106
109
|
|
110
|
+
def break_chain
|
111
|
+
info "break chain"
|
112
|
+
scheduler.clear_pending_list
|
113
|
+
@chain_breaker = true
|
114
|
+
end
|
115
|
+
|
107
116
|
def clear
|
108
117
|
@processes = Eye::Utils::AliveArray.new
|
109
118
|
end
|
data/lib/eye/group/chain.rb
CHANGED
@@ -7,6 +7,9 @@ private
|
|
7
7
|
|
8
8
|
@chain_processes_count = @processes.size
|
9
9
|
@chain_processes_current = 0
|
10
|
+
@chain_breaker = false
|
11
|
+
|
12
|
+
started_at = Time.now
|
10
13
|
|
11
14
|
@processes.each do | process |
|
12
15
|
chain_schedule_process(process, type, command, *args)
|
@@ -15,11 +18,16 @@ private
|
|
15
18
|
|
16
19
|
# to skip last sleep
|
17
20
|
break if @chain_processes_current.to_i == @chain_processes_count.to_i
|
21
|
+
break if @chain_breaker
|
18
22
|
|
19
23
|
# wait next process
|
20
24
|
sleep grace.to_f
|
25
|
+
|
26
|
+
break if @chain_breaker
|
21
27
|
end
|
22
28
|
|
29
|
+
debug "chain finished #{Time.now - started_at}s"
|
30
|
+
|
23
31
|
@chain_processes_count = nil
|
24
32
|
@chain_processes_current = nil
|
25
33
|
end
|
@@ -29,6 +37,9 @@ private
|
|
29
37
|
|
30
38
|
if type == :sync
|
31
39
|
# sync command, with waiting
|
40
|
+
# this is very hackety, because call method of the process without its scheduler
|
41
|
+
# need to provide some scheduler future
|
42
|
+
process.last_scheduled_reason = self.last_scheduled_reason
|
32
43
|
process.send(command, *args)
|
33
44
|
else
|
34
45
|
# async command
|
data/lib/eye/notify.rb
CHANGED
data/lib/eye/notify/mail.rb
CHANGED
@@ -13,7 +13,7 @@ class Eye::Notify::Mail < Eye::Notify
|
|
13
13
|
param :domain, String
|
14
14
|
param :user, String
|
15
15
|
param :password, String
|
16
|
-
param :auth, Symbol
|
16
|
+
param :auth, Symbol, nil, nil, [:plain, :login, :cram_md5]
|
17
17
|
|
18
18
|
param :from_mail, String
|
19
19
|
param :from_name, String, nil, 'eye'
|
data/lib/eye/process/child.rb
CHANGED
data/lib/eye/process/commands.rb
CHANGED
@@ -17,8 +17,8 @@ module Eye::Process::Commands
|
|
17
17
|
debug "process (#{self.pid}) ok started"
|
18
18
|
switch :started
|
19
19
|
else
|
20
|
-
|
21
|
-
if
|
20
|
+
error "process (#{self.pid}) failed to start (#{result[:error].inspect})"
|
21
|
+
if process_realy_running?
|
22
22
|
warn "kill, what remains from process (#{self.pid}), because its failed to start (without pid_file impossible to monitoring)"
|
23
23
|
send_signal(:KILL)
|
24
24
|
sleep 0.2 # little grace
|
@@ -53,7 +53,7 @@ module Eye::Process::Commands
|
|
53
53
|
switch :stopped
|
54
54
|
|
55
55
|
if control_pid?
|
56
|
-
info "delete pid_file: #{self[:
|
56
|
+
info "delete pid_file: #{self[:pid_file_ex]}"
|
57
57
|
clear_pid_file
|
58
58
|
end
|
59
59
|
|
@@ -135,7 +135,7 @@ private
|
|
135
135
|
|
136
136
|
sleep self[:stop_grace].to_f
|
137
137
|
|
138
|
-
# if process not die here, by default we kill it
|
138
|
+
# if process not die here, by default we force kill it
|
139
139
|
if process_realy_running?
|
140
140
|
warn "process not die after TERM and stop_grace #{self[:stop_grace].to_f}s, so send KILL"
|
141
141
|
send_signal(:KILL)
|
@@ -188,17 +188,14 @@ private
|
|
188
188
|
sleep self[:start_grace].to_f
|
189
189
|
|
190
190
|
unless process_realy_running?
|
191
|
-
error "process with pid
|
191
|
+
error "process with pid(#{self.pid}) not found, may be crashed (#{check_logs_str})"
|
192
192
|
return {:error => :not_realy_running}
|
193
193
|
end
|
194
194
|
|
195
|
-
|
196
|
-
|
197
|
-
rescue => ex
|
198
|
-
error "save pid to file raised with #{ex.inspect}"
|
199
|
-
return {:error => :cant_write_pid}
|
195
|
+
unless failsafe_save_pid
|
196
|
+
return {:error => :cant_write_pid}
|
200
197
|
end
|
201
|
-
|
198
|
+
|
202
199
|
res
|
203
200
|
end
|
204
201
|
|
@@ -226,17 +223,17 @@ private
|
|
226
223
|
sleep self[:start_grace].to_f
|
227
224
|
|
228
225
|
unless set_pid_from_file
|
229
|
-
error "pid_file(#{self[:
|
226
|
+
error "pid_file(#{self[:pid_file_ex]}) does not appears after start_grace #{self[:start_grace].to_f}, check start_command, or tune start_grace (eye dont know what to monitor without pid)"
|
230
227
|
return {:error => :pid_not_found}
|
231
228
|
end
|
232
229
|
|
233
230
|
unless process_realy_running?
|
234
|
-
error "process in pid_file(#{self[:
|
231
|
+
error "process in pid_file(#{self[:pid_file_ex]})(#{self.pid}) not found, maybe process do not write there actual pid, or just crashed (#{check_logs_str})"
|
235
232
|
return {:error => :not_realy_running}
|
236
233
|
end
|
237
234
|
|
238
235
|
res[:pid] = self.pid
|
239
|
-
info "process get pid:#{res[:pid]}"
|
236
|
+
info "process get pid:#{res[:pid]}, pid_file #{self[:pid_file_ex]}"
|
240
237
|
res
|
241
238
|
end
|
242
239
|
|
data/lib/eye/process/monitor.rb
CHANGED
@@ -38,7 +38,7 @@ private
|
|
38
38
|
|
39
39
|
# check that process runned
|
40
40
|
unless process_realy_running?
|
41
|
-
warn
|
41
|
+
warn "check_alive: process(#{self.pid}) not found!"
|
42
42
|
notify :warn, 'crashed!'
|
43
43
|
switch :crashed
|
44
44
|
else
|
@@ -49,14 +49,20 @@ private
|
|
49
49
|
msg = "check_alive: pid_file(#{self[:pid_file]}) changes by itself (pid:#{self.pid}) => (pid:#{ppid})"
|
50
50
|
if control_pid?
|
51
51
|
msg += ", not correct, pid_file is under eye control, so rewrited back pid:#{self.pid}"
|
52
|
-
|
52
|
+
unless failsafe_save_pid
|
53
|
+
msg += ', (Can`t rewrite pid_file O_o)'
|
54
|
+
end
|
53
55
|
else
|
54
56
|
if ppid == nil
|
55
57
|
msg += ', rewrited because empty'
|
56
|
-
|
58
|
+
unless failsafe_save_pid
|
59
|
+
msg += ', (Can`t rewrite pid_file O_o)'
|
60
|
+
end
|
57
61
|
elsif (Time.now - pid_file_ctime > REWRITE_FACKUP_PIDFILE_PERIOD)
|
58
62
|
msg += ", > #{REWRITE_FACKUP_PIDFILE_PERIOD.inspect} ago, so rewrited (even if pid_file not under eye control)"
|
59
|
-
|
63
|
+
unless failsafe_save_pid
|
64
|
+
msg += ', (Can`t rewrite pid_file O_o)'
|
65
|
+
end
|
60
66
|
else
|
61
67
|
msg += ', not under eye control, so ignored'
|
62
68
|
end
|
@@ -68,18 +74,6 @@ private
|
|
68
74
|
end
|
69
75
|
end
|
70
76
|
|
71
|
-
def failsafe_load_pid
|
72
|
-
pid = load_pid_from_file
|
73
|
-
|
74
|
-
if !pid
|
75
|
-
# this is can be symlink changed case
|
76
|
-
sleep 0.1
|
77
|
-
pid = load_pid_from_file
|
78
|
-
end
|
79
|
-
|
80
|
-
pid
|
81
|
-
end
|
82
|
-
|
83
77
|
def check_crash
|
84
78
|
if down?
|
85
79
|
if self[:keep_alive] && !@flapping
|
@@ -3,18 +3,23 @@ module Eye::Process::Scheduler
|
|
3
3
|
# ex: schedule :update_config, config, "reason: update_config"
|
4
4
|
def schedule(command, *args, &block)
|
5
5
|
if scheduler.alive?
|
6
|
+
unless self.respond_to?(command)
|
7
|
+
warn "object not support :#{command} to schedule"
|
8
|
+
return
|
9
|
+
end
|
10
|
+
|
6
11
|
reason = if args.present? && [String, Symbol].include?(args[-1].class)
|
7
12
|
args.pop
|
8
13
|
end
|
9
14
|
|
10
|
-
info "schedule :#{command} (#{reason})"
|
15
|
+
info "schedule :#{command} #{reason ? "(reason: #{reason})" : nil}"
|
11
16
|
scheduler.add_wo_dups(:scheduled_action, command, {:args => args, :reason => reason}, &block)
|
12
17
|
end
|
13
18
|
end
|
14
19
|
|
15
20
|
def scheduled_action(command, h = {}, &block)
|
16
21
|
reason = h.delete(:reason)
|
17
|
-
info "=> #{command} #{h[:args].present? ? "#{h[:args]*',' }" : nil}(#{reason})"
|
22
|
+
info "=> #{command} #{h[:args].present? ? "#{h[:args]*',' }" : nil} #{reason ? "(reason: #{reason})" : nil}"
|
18
23
|
|
19
24
|
@current_scheduled_command = command
|
20
25
|
@last_scheduled_command = command
|
@@ -47,4 +52,4 @@ private
|
|
47
52
|
@scheduler ||= Eye::Utils::CelluloidChain.new(current_actor)
|
48
53
|
end
|
49
54
|
|
50
|
-
end
|
55
|
+
end
|
data/lib/eye/process/states.rb
CHANGED
@@ -1,4 +1,3 @@
|
|
1
|
-
gem 'state_machine'
|
2
1
|
require 'state_machine'
|
3
2
|
|
4
3
|
class Eye::Process
|
@@ -82,7 +81,7 @@ class Eye::Process
|
|
82
81
|
|
83
82
|
def log_transition(transition)
|
84
83
|
@states_history.push transition.to_name, @state_reason
|
85
|
-
info "switch :#{transition.event} [:#{transition.from_name} => :#{transition.to_name}] #{@state_reason ? "(#{@state_reason})" : nil}"
|
84
|
+
info "switch :#{transition.event} [:#{transition.from_name} => :#{transition.to_name}] #{@state_reason ? "(reason: #{@state_reason})" : nil}"
|
86
85
|
end
|
87
86
|
|
88
87
|
def upd_for_triggers(transition)
|
data/lib/eye/process/system.rb
CHANGED
@@ -18,9 +18,10 @@ module Eye::Process::System
|
|
18
18
|
File.open(self[:pid_file_ex], 'w') do |f|
|
19
19
|
f.write self.pid
|
20
20
|
end
|
21
|
+
true
|
22
|
+
else
|
23
|
+
false
|
21
24
|
end
|
22
|
-
|
23
|
-
true
|
24
25
|
end
|
25
26
|
|
26
27
|
def clear_pid_file
|
@@ -35,16 +36,26 @@ module Eye::Process::System
|
|
35
36
|
end
|
36
37
|
|
37
38
|
def process_realy_running?
|
38
|
-
|
39
|
+
if self.pid
|
40
|
+
res = Eye::System.check_pid_alive(self.pid)
|
41
|
+
if res[:error] && res[:error].class != Errno::ESRCH
|
42
|
+
error "process_realy_running?: check_pid_alive returns '#{res[:error].message}'"
|
43
|
+
end
|
44
|
+
res[:result]
|
45
|
+
else
|
46
|
+
debug "process_realy_running?: called without pid"
|
47
|
+
nil
|
48
|
+
end
|
39
49
|
end
|
40
50
|
|
41
51
|
def send_signal(code)
|
42
|
-
debug "send signal #{code}"
|
43
|
-
|
44
52
|
res = Eye::System.send_signal(self.pid, code)
|
45
|
-
error(res[:message]) if res[:status] != :ok
|
46
53
|
|
47
|
-
|
54
|
+
msg = "send_signal #{code} to #{self.pid}"
|
55
|
+
msg += ", error<#{res[:error]}>" if res[:error]
|
56
|
+
info msg
|
57
|
+
|
58
|
+
res[:result] == :ok
|
48
59
|
end
|
49
60
|
|
50
61
|
# non blocking actor timeout
|
@@ -56,6 +67,26 @@ module Eye::Process::System
|
|
56
67
|
defer{ Eye::System::execute cmd, cfg }
|
57
68
|
end
|
58
69
|
|
70
|
+
def failsafe_load_pid
|
71
|
+
pid = load_pid_from_file
|
72
|
+
|
73
|
+
if !pid
|
74
|
+
# this is can be symlink changed case
|
75
|
+
sleep 0.1
|
76
|
+
pid = load_pid_from_file
|
77
|
+
end
|
78
|
+
|
79
|
+
pid
|
80
|
+
end
|
81
|
+
|
82
|
+
def failsafe_save_pid
|
83
|
+
save_pid_to_file
|
84
|
+
true
|
85
|
+
rescue => ex
|
86
|
+
error "failsafe_save_pid: #{ex.message}"
|
87
|
+
false
|
88
|
+
end
|
89
|
+
|
59
90
|
private
|
60
91
|
|
61
92
|
def wait_for_condition_sync(timeout, step, &block)
|