eye 0.3.2 → 0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.gitignore +1 -0
- data/.rspec +1 -1
- data/.travis.yml +3 -1
- data/CHANGES.md +11 -2
- data/Gemfile +1 -0
- data/README.md +18 -14
- data/Rakefile +10 -3
- data/bin/eye +41 -27
- data/examples/process_thin.rb +1 -1
- data/examples/processes/em.rb +2 -2
- data/examples/processes/forking.rb +2 -2
- data/examples/processes/sample.rb +5 -5
- data/examples/rbenv.eye +1 -1
- data/examples/sidekiq.eye +2 -2
- data/examples/test.eye +10 -6
- data/examples/thin-farm.eye +1 -1
- data/examples/unicorn.eye +1 -1
- data/eye.gemspec +13 -7
- data/lib/eye.rb +6 -6
- data/lib/eye/application.rb +9 -6
- data/lib/eye/checker.rb +51 -21
- data/lib/eye/checker/file_size.rb +1 -1
- data/lib/eye/checker/http.rb +3 -3
- data/lib/eye/checker/memory.rb +1 -1
- data/lib/eye/checker/socket.rb +6 -6
- data/lib/eye/child_process.rb +7 -11
- data/lib/eye/client.rb +6 -6
- data/lib/eye/config.rb +2 -2
- data/lib/eye/controller.rb +11 -8
- data/lib/eye/controller/commands.rb +8 -9
- data/lib/eye/controller/helpers.rb +1 -0
- data/lib/eye/controller/load.rb +11 -7
- data/lib/eye/controller/send_command.rb +44 -19
- data/lib/eye/controller/show_history.rb +8 -7
- data/lib/eye/controller/status.rb +39 -26
- data/lib/eye/dsl.rb +3 -3
- data/lib/eye/dsl/application_opts.rb +4 -4
- data/lib/eye/dsl/config_opts.rb +4 -4
- data/lib/eye/dsl/helpers.rb +2 -2
- data/lib/eye/dsl/main.rb +2 -2
- data/lib/eye/dsl/opts.rb +19 -14
- data/lib/eye/dsl/process_opts.rb +1 -1
- data/lib/eye/dsl/pure_opts.rb +2 -2
- data/lib/eye/dsl/validation.rb +7 -5
- data/lib/eye/group.rb +17 -11
- data/lib/eye/group/chain.rb +3 -3
- data/lib/eye/loader.rb +8 -6
- data/lib/eye/logger.rb +14 -5
- data/lib/eye/notify.rb +13 -7
- data/lib/eye/notify/jabber.rb +2 -2
- data/lib/eye/notify/mail.rb +2 -2
- data/lib/eye/process.rb +10 -13
- data/lib/eye/process/child.rb +1 -1
- data/lib/eye/process/commands.rb +34 -32
- data/lib/eye/process/config.rb +17 -12
- data/lib/eye/process/controller.rb +3 -6
- data/lib/eye/process/data.rb +16 -5
- data/lib/eye/process/monitor.rb +12 -5
- data/lib/eye/process/notify.rb +1 -1
- data/lib/eye/process/scheduler.rb +3 -3
- data/lib/eye/process/states.rb +10 -13
- data/lib/eye/process/states_history.rb +3 -3
- data/lib/eye/process/system.rb +17 -21
- data/lib/eye/process/trigger.rb +11 -30
- data/lib/eye/process/watchers.rb +9 -9
- data/lib/eye/server.rb +14 -6
- data/lib/eye/settings.rb +4 -4
- data/lib/eye/system.rb +10 -7
- data/lib/eye/system_resources.rb +4 -4
- data/lib/eye/trigger.rb +58 -21
- data/lib/eye/trigger/flapping.rb +24 -4
- data/lib/eye/trigger/state.rb +28 -0
- data/lib/eye/utils/alive_array.rb +1 -1
- data/lib/eye/utils/celluloid_klass.rb +5 -0
- data/lib/eye/utils/pmap.rb +7 -0
- data/lib/eye/utils/tail.rb +1 -1
- metadata +39 -23
- data/lib/eye/utils/leak_19.rb +0 -7
data/lib/eye/process/notify.rb
CHANGED
@@ -13,7 +13,7 @@ module Eye::Process::Notify
|
|
13
13
|
|
14
14
|
# send notifies
|
15
15
|
if self[:notify].present?
|
16
|
-
message = {:message => msg, :name => name,
|
16
|
+
message = {:message => msg, :name => name,
|
17
17
|
:full_name => full_name, :pid => pid, :host => Eye::System.host, :level => level,
|
18
18
|
:at => Time.now }
|
19
19
|
|
@@ -17,7 +17,7 @@ module Eye::Process::Scheduler
|
|
17
17
|
if reason.class == Eye::Reason
|
18
18
|
# for auto reasons
|
19
19
|
# skip already running commands and all in chain
|
20
|
-
scheduler.add_wo_dups_current(:scheduled_action, command, {:args => args, :reason => reason}, &block)
|
20
|
+
scheduler.add_wo_dups_current(:scheduled_action, command, {:args => args, :reason => reason}, &block)
|
21
21
|
else
|
22
22
|
# for manual, or without reason
|
23
23
|
# skip only for last in chain
|
@@ -57,7 +57,7 @@ module Eye::Process::Scheduler
|
|
57
57
|
def scheduler_clear_pending_list
|
58
58
|
scheduler.clear_pending_list
|
59
59
|
end
|
60
|
-
|
60
|
+
|
61
61
|
def self.included(base)
|
62
62
|
base.finalizer :remove_scheduler
|
63
63
|
end
|
@@ -74,7 +74,7 @@ private
|
|
74
74
|
def remove_scheduler
|
75
75
|
@scheduler.terminate if @scheduler && @scheduler.alive?
|
76
76
|
end
|
77
|
-
|
77
|
+
|
78
78
|
def scheduler
|
79
79
|
@scheduler ||= Eye::Utils::CelluloidChain.new(current_actor)
|
80
80
|
end
|
data/lib/eye/process/states.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'state_machine'
|
2
|
+
require 'state_machine/version'
|
2
3
|
|
3
4
|
class Eye::Process
|
4
5
|
|
@@ -52,11 +53,17 @@ class Eye::Process
|
|
52
53
|
transition any => :unmonitored
|
53
54
|
end
|
54
55
|
|
55
|
-
after_transition any => :unmonitored, :do => :on_unmonitored
|
56
|
-
after_transition any-:up => :up, :do => :on_up
|
57
|
-
after_transition :up => any-:up, :do => :from_up
|
58
56
|
after_transition any => any, :do => :log_transition
|
59
57
|
after_transition any => any, :do => :check_triggers
|
58
|
+
|
59
|
+
after_transition any => :unmonitored, :do => :on_unmonitored
|
60
|
+
|
61
|
+
after_transition any-:up => :up, :do => :add_watchers
|
62
|
+
after_transition :up => any-:up, :do => :remove_watchers
|
63
|
+
|
64
|
+
after_transition any-:up => :up, :do => :add_childs
|
65
|
+
after_transition any => [:unmonitored, :down], :do => :remove_childs
|
66
|
+
|
60
67
|
after_transition :on => :crashed, :do => :on_crashed
|
61
68
|
end
|
62
69
|
|
@@ -68,16 +75,6 @@ class Eye::Process
|
|
68
75
|
self.pid = nil
|
69
76
|
end
|
70
77
|
|
71
|
-
def on_up
|
72
|
-
add_watchers
|
73
|
-
add_childs
|
74
|
-
end
|
75
|
-
|
76
|
-
def from_up
|
77
|
-
remove_watchers
|
78
|
-
remove_childs
|
79
|
-
end
|
80
|
-
|
81
78
|
def log_transition(transition)
|
82
79
|
@states_history.push transition.to_name, @state_reason
|
83
80
|
info "switch :#{transition.event} [:#{transition.from_name} => :#{transition.to_name}] #{@state_reason ? "(reason: #{@state_reason})" : nil}"
|
@@ -40,19 +40,19 @@ class Eye::Process::StatesHistory < Eye::Utils::Tail
|
|
40
40
|
end
|
41
41
|
|
42
42
|
def any?(*seq)
|
43
|
-
states.any? do |st|
|
43
|
+
states.any? do |st|
|
44
44
|
seq.flatten.include?(st)
|
45
45
|
end
|
46
46
|
end
|
47
47
|
|
48
48
|
def noone?(*seq)
|
49
|
-
!states.all? do |st|
|
49
|
+
!states.all? do |st|
|
50
50
|
seq.flatten.include?(st)
|
51
51
|
end
|
52
52
|
end
|
53
53
|
|
54
54
|
def all?(*seq)
|
55
|
-
states.all? do |st|
|
55
|
+
states.all? do |st|
|
56
56
|
seq.flatten.include?(st)
|
57
57
|
end
|
58
58
|
end
|
data/lib/eye/process/system.rb
CHANGED
@@ -2,9 +2,9 @@ require 'timeout'
|
|
2
2
|
|
3
3
|
module Eye::Process::System
|
4
4
|
|
5
|
-
def load_pid_from_file
|
5
|
+
def load_pid_from_file
|
6
6
|
if File.exists?(self[:pid_file_ex])
|
7
|
-
_pid = File.read(self[:pid_file_ex]).to_i
|
7
|
+
_pid = File.read(self[:pid_file_ex]).to_i
|
8
8
|
_pid > 0 ? _pid : nil
|
9
9
|
end
|
10
10
|
end
|
@@ -25,9 +25,10 @@ module Eye::Process::System
|
|
25
25
|
end
|
26
26
|
|
27
27
|
def clear_pid_file
|
28
|
+
info "delete pid_file: #{self[:pid_file_ex]}"
|
28
29
|
File.unlink(self[:pid_file_ex])
|
29
30
|
true
|
30
|
-
rescue
|
31
|
+
rescue
|
31
32
|
nil
|
32
33
|
end
|
33
34
|
|
@@ -51,9 +52,18 @@ module Eye::Process::System
|
|
51
52
|
res[:result] == :ok
|
52
53
|
end
|
53
54
|
|
54
|
-
# non blocking actor timeout
|
55
55
|
def wait_for_condition(timeout, step = 0.1, &block)
|
56
|
-
|
56
|
+
res = nil
|
57
|
+
sumtime = 0
|
58
|
+
|
59
|
+
loop do
|
60
|
+
tm = Time.now
|
61
|
+
res = yield # note that yield can block actor here and timeout can be overhead
|
62
|
+
return res if res
|
63
|
+
sleep step.to_f
|
64
|
+
sumtime += (Time.now - tm)
|
65
|
+
return false if sumtime > timeout
|
66
|
+
end
|
57
67
|
end
|
58
68
|
|
59
69
|
def execute(cmd, cfg = {})
|
@@ -63,7 +73,7 @@ module Eye::Process::System
|
|
63
73
|
def failsafe_load_pid
|
64
74
|
pid = load_pid_from_file
|
65
75
|
|
66
|
-
if !pid
|
76
|
+
if !pid
|
67
77
|
# this is can be symlink changed case
|
68
78
|
sleep 0.1
|
69
79
|
pid = load_pid_from_file
|
@@ -73,25 +83,11 @@ module Eye::Process::System
|
|
73
83
|
end
|
74
84
|
|
75
85
|
def failsafe_save_pid
|
76
|
-
save_pid_to_file
|
86
|
+
save_pid_to_file
|
77
87
|
true
|
78
88
|
rescue => ex
|
79
89
|
error "failsafe_save_pid: #{ex.message}"
|
80
90
|
false
|
81
91
|
end
|
82
92
|
|
83
|
-
private
|
84
|
-
|
85
|
-
def wait_for_condition_sync(timeout, step, &block)
|
86
|
-
res = nil
|
87
|
-
|
88
|
-
Timeout::timeout(timeout.to_f) do
|
89
|
-
sleep step.to_f until res = yield
|
90
|
-
end
|
91
|
-
|
92
|
-
res
|
93
|
-
rescue Timeout::Error
|
94
|
-
false
|
95
|
-
end
|
96
|
-
|
97
93
|
end
|
data/lib/eye/process/trigger.rb
CHANGED
@@ -4,7 +4,7 @@ module Eye::Process::Trigger
|
|
4
4
|
if self[:triggers]
|
5
5
|
self[:triggers].each do |type, cfg|
|
6
6
|
add_trigger(cfg)
|
7
|
-
end
|
7
|
+
end
|
8
8
|
end
|
9
9
|
end
|
10
10
|
|
@@ -12,43 +12,24 @@ module Eye::Process::Trigger
|
|
12
12
|
self.triggers = []
|
13
13
|
end
|
14
14
|
|
15
|
-
def check_triggers
|
15
|
+
def check_triggers(transition)
|
16
16
|
return if unmonitored?
|
17
|
+
self.triggers.each { |trigger| trigger.notify(transition) }
|
18
|
+
end
|
17
19
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
20
|
+
def retry_start_after_flapping
|
21
|
+
return unless unmonitored?
|
22
|
+
return unless state_reason.to_s.include?('flapping') # TODO: remove hackety
|
23
|
+
|
24
|
+
schedule :start, Eye::Reason.new(:'retry start after flapping')
|
25
|
+
self.flapping_times += 1
|
23
26
|
end
|
24
27
|
|
25
28
|
private
|
26
29
|
|
27
30
|
def add_trigger(cfg = {})
|
28
|
-
trigger = Eye::Trigger.create(
|
31
|
+
trigger = Eye::Trigger.create(current_actor, cfg)
|
29
32
|
self.triggers << trigger
|
30
33
|
end
|
31
34
|
|
32
|
-
def on_flapping(trigger)
|
33
|
-
notify :error, 'flapping!'
|
34
|
-
schedule :unmonitor, Eye::Reason.new(:flapping)
|
35
|
-
|
36
|
-
@retry_times ||= 0
|
37
|
-
retry_in = trigger.retry_in
|
38
|
-
|
39
|
-
return unless retry_in
|
40
|
-
return if trigger.retry_times && @retry_times >= trigger.retry_times
|
41
|
-
|
42
|
-
schedule_in(retry_in.to_f, :retry_action)
|
43
|
-
end
|
44
|
-
|
45
|
-
def retry_action
|
46
|
-
debug "trigger retry timer"
|
47
|
-
return unless unmonitored?
|
48
|
-
return unless state_reason.to_s.include?('flapping') # TODO: remove hackety
|
49
|
-
|
50
|
-
schedule :start, Eye::Reason.new(:'retry start after flapping')
|
51
|
-
@retry_times += 1
|
52
|
-
end
|
53
|
-
|
54
35
|
end
|
data/lib/eye/process/watchers.rb
CHANGED
@@ -7,13 +7,13 @@ module Eye::Process::Watchers
|
|
7
7
|
|
8
8
|
if @watchers.blank?
|
9
9
|
# default watcher :check_alive
|
10
|
-
add_watcher(:check_alive, self[:check_alive_period]) do
|
10
|
+
add_watcher(:check_alive, self[:check_alive_period]) do
|
11
11
|
check_alive
|
12
12
|
end
|
13
13
|
|
14
14
|
# monitor childs pids
|
15
15
|
if self[:monitor_children]
|
16
|
-
add_watcher(:check_childs, self[:childs_update_period]) do
|
16
|
+
add_watcher(:check_childs, self[:childs_update_period]) do
|
17
17
|
add_or_update_childs
|
18
18
|
end
|
19
19
|
end
|
@@ -24,13 +24,13 @@ module Eye::Process::Watchers
|
|
24
24
|
warn 'try add_watchers, but its already here'
|
25
25
|
end
|
26
26
|
end
|
27
|
-
|
27
|
+
|
28
28
|
def remove_watchers
|
29
29
|
@watchers.each{|_, h| h[:timer].cancel }
|
30
30
|
@watchers = {}
|
31
31
|
end
|
32
32
|
|
33
|
-
private
|
33
|
+
private
|
34
34
|
|
35
35
|
def add_watcher(type, period = 2, subject = nil, &block)
|
36
36
|
return if @watchers[type]
|
@@ -40,17 +40,17 @@ private
|
|
40
40
|
timer = every(period.to_f) do
|
41
41
|
debug "check #{type}"
|
42
42
|
block.call(subject)
|
43
|
-
end
|
43
|
+
end
|
44
44
|
|
45
45
|
@watchers[type] ||= {:timer => timer, :subject => subject}
|
46
|
-
end
|
47
|
-
|
46
|
+
end
|
47
|
+
|
48
48
|
def start_checkers
|
49
49
|
self[:checks].each{|name, cfg| start_checker(name, cfg) }
|
50
50
|
end
|
51
51
|
|
52
52
|
def start_checker(name, cfg)
|
53
|
-
subject = Eye::Checker.create(pid, cfg,
|
53
|
+
subject = Eye::Checker.create(pid, cfg, current_actor)
|
54
54
|
|
55
55
|
# ex: {:type => :memory, :every => 5.seconds, :below => 100.megabytes, :times => [3,5]}
|
56
56
|
add_watcher("check_#{name}".to_sym, subject.every, subject, &method(:watcher_tick).to_proc)
|
@@ -59,7 +59,7 @@ private
|
|
59
59
|
def watcher_tick(subject)
|
60
60
|
unless subject.check
|
61
61
|
return unless up?
|
62
|
-
|
62
|
+
|
63
63
|
action = subject.fire || :restart
|
64
64
|
notify :warn, "Bounded #{subject.check_name}: #{subject.last_human_values} send to :#{action}"
|
65
65
|
schedule action, Eye::Reason.new("bounded #{subject.check_name}")
|
data/lib/eye/server.rb
CHANGED
@@ -3,9 +3,9 @@ require 'celluloid/autostart'
|
|
3
3
|
|
4
4
|
class Eye::Server
|
5
5
|
include Celluloid::IO
|
6
|
-
|
6
|
+
|
7
7
|
attr_reader :socket_path, :server
|
8
|
-
|
8
|
+
|
9
9
|
def initialize(socket_path)
|
10
10
|
@socket_path = socket_path
|
11
11
|
@server = begin
|
@@ -21,14 +21,22 @@ class Eye::Server
|
|
21
21
|
end
|
22
22
|
|
23
23
|
def handle_connection(socket)
|
24
|
-
|
24
|
+
text = socket.read
|
25
|
+
|
26
|
+
begin
|
27
|
+
command, *args = Marshal.load(text)
|
28
|
+
rescue => ex
|
29
|
+
error "Failed socket read #{ex.message}"
|
30
|
+
return
|
31
|
+
end
|
32
|
+
|
25
33
|
response = command(command, *args)
|
26
34
|
socket.write(Marshal.dump(response))
|
27
|
-
|
35
|
+
|
28
36
|
rescue Errno::EPIPE
|
29
37
|
# client timeouted
|
30
38
|
# do nothing
|
31
|
-
|
39
|
+
|
32
40
|
ensure
|
33
41
|
socket.close
|
34
42
|
end
|
@@ -43,7 +51,7 @@ class Eye::Server
|
|
43
51
|
end
|
44
52
|
|
45
53
|
finalizer :close_socket
|
46
|
-
|
54
|
+
|
47
55
|
def close_socket
|
48
56
|
@server.close if @server
|
49
57
|
unlink_socket_file
|
data/lib/eye/settings.rb
CHANGED
@@ -2,7 +2,7 @@ require 'fileutils'
|
|
2
2
|
|
3
3
|
module Eye::Settings
|
4
4
|
module_function
|
5
|
-
|
5
|
+
|
6
6
|
def dir
|
7
7
|
if root?
|
8
8
|
'/var/run/eye'
|
@@ -26,7 +26,7 @@ module Eye::Settings
|
|
26
26
|
def home
|
27
27
|
ENV['EYE_HOME'] || ENV['HOME']
|
28
28
|
end
|
29
|
-
|
29
|
+
|
30
30
|
def path(path)
|
31
31
|
File.join(dir, path)
|
32
32
|
end
|
@@ -38,11 +38,11 @@ module Eye::Settings
|
|
38
38
|
def socket_path
|
39
39
|
path(ENV['EYE_SOCK'] || "sock#{ENV['EYE_V']}")
|
40
40
|
end
|
41
|
-
|
41
|
+
|
42
42
|
def pid_path
|
43
43
|
path(ENV['EYE_PID'] || "pid#{ENV['EYE_V']}")
|
44
44
|
end
|
45
|
-
|
45
|
+
|
46
46
|
def cache_path
|
47
47
|
path("processes#{ENV['EYE_V']}.cache")
|
48
48
|
end
|
data/lib/eye/system.rb
CHANGED
@@ -29,7 +29,7 @@ module Eye::System
|
|
29
29
|
|
30
30
|
# Send signal to process (uses for kill)
|
31
31
|
# code: TERM(15), KILL(9), QUIT(3), ...
|
32
|
-
def send_signal(pid, code = :TERM)
|
32
|
+
def send_signal(pid, code = :TERM)
|
33
33
|
code = 0 if code == '0'
|
34
34
|
if code.to_s.to_i != 0
|
35
35
|
code = code.to_i
|
@@ -38,7 +38,7 @@ module Eye::System
|
|
38
38
|
code = code.to_s.upcase if code.is_a?(String) || code.is_a?(Symbol)
|
39
39
|
|
40
40
|
if pid
|
41
|
-
::Process.kill(code, pid)
|
41
|
+
::Process.kill(code, pid)
|
42
42
|
{:result => :ok}
|
43
43
|
else
|
44
44
|
{:error => Exception.new('no_pid')}
|
@@ -59,7 +59,7 @@ module Eye::System
|
|
59
59
|
pid = Process::spawn(prepare_env(cfg), *Shellwords.shellwords(cmd), opts)
|
60
60
|
Process.detach(pid)
|
61
61
|
{:pid => pid}
|
62
|
-
|
62
|
+
|
63
63
|
rescue Errno::ENOENT, Errno::EACCES => ex
|
64
64
|
{:error => ex}
|
65
65
|
end
|
@@ -82,7 +82,7 @@ module Eye::System
|
|
82
82
|
|
83
83
|
rescue Timeout::Error => ex
|
84
84
|
if pid
|
85
|
-
|
85
|
+
warn "[#{cfg[:name]}] send signal 9 to #{pid} (because of timeouted<#{timeout}> execution)"
|
86
86
|
send_signal(pid, 9)
|
87
87
|
end
|
88
88
|
{:error => ex}
|
@@ -90,7 +90,7 @@ module Eye::System
|
|
90
90
|
rescue Errno::ENOENT, Errno::EACCES => ex
|
91
91
|
{:error => ex}
|
92
92
|
|
93
|
-
ensure
|
93
|
+
ensure
|
94
94
|
Process.detach(pid) if pid
|
95
95
|
end
|
96
96
|
|
@@ -102,7 +102,7 @@ module Eye::System
|
|
102
102
|
h = {}
|
103
103
|
str.each_line do |line|
|
104
104
|
chunk = line.strip.split(/\s+/)
|
105
|
-
h[chunk[0].to_i] = { :ppid => chunk[1].to_i, :cpu => chunk[2].to_i,
|
105
|
+
h[chunk[0].to_i] = { :ppid => chunk[1].to_i, :cpu => chunk[2].to_i,
|
106
106
|
:rss => chunk[3].to_i, :start_time => chunk[4] }
|
107
107
|
end
|
108
108
|
h
|
@@ -114,7 +114,10 @@ module Eye::System
|
|
114
114
|
end
|
115
115
|
|
116
116
|
def host
|
117
|
-
@host ||=
|
117
|
+
@host ||= begin
|
118
|
+
require 'socket'
|
119
|
+
Socket.gethostname
|
120
|
+
end
|
118
121
|
end
|
119
122
|
|
120
123
|
# set host for tests
|