reel-eye 0.3.2 → 0.4
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +1 -0
- data/.rspec +1 -1
- data/.travis.yml +3 -1
- data/CHANGES.md +11 -2
- data/Gemfile +1 -0
- data/README.md +18 -14
- data/Rakefile +10 -3
- data/bin/eye +41 -27
- data/examples/process_thin.rb +1 -1
- data/examples/processes/em.rb +2 -2
- data/examples/processes/forking.rb +2 -2
- data/examples/processes/sample.rb +5 -5
- data/examples/rbenv.eye +1 -1
- data/examples/sidekiq.eye +2 -2
- data/examples/test.eye +10 -6
- data/examples/thin-farm.eye +1 -1
- data/examples/unicorn.eye +1 -1
- data/eye.gemspec +13 -7
- data/lib/eye.rb +6 -6
- data/lib/eye/application.rb +9 -6
- data/lib/eye/checker.rb +51 -21
- data/lib/eye/checker/file_size.rb +1 -1
- data/lib/eye/checker/http.rb +3 -3
- data/lib/eye/checker/memory.rb +1 -1
- data/lib/eye/checker/socket.rb +6 -6
- data/lib/eye/child_process.rb +7 -11
- data/lib/eye/client.rb +6 -6
- data/lib/eye/config.rb +2 -2
- data/lib/eye/controller.rb +13 -8
- data/lib/eye/controller/commands.rb +9 -10
- data/lib/eye/controller/helpers.rb +1 -0
- data/lib/eye/controller/load.rb +11 -7
- data/lib/eye/controller/options.rb +1 -1
- data/lib/eye/controller/send_command.rb +44 -19
- data/lib/eye/controller/show_history.rb +8 -7
- data/lib/eye/controller/status.rb +39 -26
- data/lib/eye/dsl.rb +3 -3
- data/lib/eye/dsl/application_opts.rb +4 -4
- data/lib/eye/dsl/config_opts.rb +4 -4
- data/lib/eye/dsl/helpers.rb +2 -2
- data/lib/eye/dsl/main.rb +2 -2
- data/lib/eye/dsl/opts.rb +19 -14
- data/lib/eye/dsl/process_opts.rb +1 -1
- data/lib/eye/dsl/pure_opts.rb +2 -2
- data/lib/eye/dsl/validation.rb +7 -5
- data/lib/eye/group.rb +17 -11
- data/lib/eye/group/chain.rb +3 -3
- data/lib/eye/http.rb +1 -1
- data/lib/eye/http/router.rb +2 -2
- data/lib/eye/loader.rb +8 -7
- data/lib/eye/logger.rb +14 -5
- data/lib/eye/notify.rb +13 -7
- data/lib/eye/notify/jabber.rb +2 -2
- data/lib/eye/notify/mail.rb +2 -2
- data/lib/eye/process.rb +10 -13
- data/lib/eye/process/child.rb +1 -1
- data/lib/eye/process/commands.rb +34 -32
- data/lib/eye/process/config.rb +17 -12
- data/lib/eye/process/controller.rb +3 -6
- data/lib/eye/process/data.rb +16 -5
- data/lib/eye/process/monitor.rb +12 -5
- data/lib/eye/process/notify.rb +1 -1
- data/lib/eye/process/scheduler.rb +3 -3
- data/lib/eye/process/states.rb +10 -13
- data/lib/eye/process/states_history.rb +3 -3
- data/lib/eye/process/system.rb +17 -21
- data/lib/eye/process/trigger.rb +11 -30
- data/lib/eye/process/watchers.rb +9 -9
- data/lib/eye/server.rb +14 -6
- data/lib/eye/settings.rb +4 -4
- data/lib/eye/system.rb +10 -7
- data/lib/eye/system_resources.rb +4 -4
- data/lib/eye/trigger.rb +58 -21
- data/lib/eye/trigger/flapping.rb +24 -4
- data/lib/eye/trigger/state.rb +28 -0
- data/lib/eye/utils/alive_array.rb +1 -1
- data/lib/eye/utils/celluloid_klass.rb +5 -0
- data/lib/eye/utils/pmap.rb +7 -0
- data/lib/eye/utils/tail.rb +1 -1
- metadata +252 -271
- data/lib/eye/utils/leak_19.rb +0 -7
data/lib/eye/process/notify.rb
CHANGED
@@ -13,7 +13,7 @@ module Eye::Process::Notify
|
|
13
13
|
|
14
14
|
# send notifies
|
15
15
|
if self[:notify].present?
|
16
|
-
message = {:message => msg, :name => name,
|
16
|
+
message = {:message => msg, :name => name,
|
17
17
|
:full_name => full_name, :pid => pid, :host => Eye::System.host, :level => level,
|
18
18
|
:at => Time.now }
|
19
19
|
|
@@ -17,7 +17,7 @@ module Eye::Process::Scheduler
|
|
17
17
|
if reason.class == Eye::Reason
|
18
18
|
# for auto reasons
|
19
19
|
# skip already running commands and all in chain
|
20
|
-
scheduler.add_wo_dups_current(:scheduled_action, command, {:args => args, :reason => reason}, &block)
|
20
|
+
scheduler.add_wo_dups_current(:scheduled_action, command, {:args => args, :reason => reason}, &block)
|
21
21
|
else
|
22
22
|
# for manual, or without reason
|
23
23
|
# skip only for last in chain
|
@@ -57,7 +57,7 @@ module Eye::Process::Scheduler
|
|
57
57
|
def scheduler_clear_pending_list
|
58
58
|
scheduler.clear_pending_list
|
59
59
|
end
|
60
|
-
|
60
|
+
|
61
61
|
def self.included(base)
|
62
62
|
base.finalizer :remove_scheduler
|
63
63
|
end
|
@@ -74,7 +74,7 @@ private
|
|
74
74
|
def remove_scheduler
|
75
75
|
@scheduler.terminate if @scheduler && @scheduler.alive?
|
76
76
|
end
|
77
|
-
|
77
|
+
|
78
78
|
def scheduler
|
79
79
|
@scheduler ||= Eye::Utils::CelluloidChain.new(current_actor)
|
80
80
|
end
|
data/lib/eye/process/states.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
require 'state_machine'
|
2
|
+
require 'state_machine/version'
|
2
3
|
|
3
4
|
class Eye::Process
|
4
5
|
|
@@ -52,11 +53,17 @@ class Eye::Process
|
|
52
53
|
transition any => :unmonitored
|
53
54
|
end
|
54
55
|
|
55
|
-
after_transition any => :unmonitored, :do => :on_unmonitored
|
56
|
-
after_transition any-:up => :up, :do => :on_up
|
57
|
-
after_transition :up => any-:up, :do => :from_up
|
58
56
|
after_transition any => any, :do => :log_transition
|
59
57
|
after_transition any => any, :do => :check_triggers
|
58
|
+
|
59
|
+
after_transition any => :unmonitored, :do => :on_unmonitored
|
60
|
+
|
61
|
+
after_transition any-:up => :up, :do => :add_watchers
|
62
|
+
after_transition :up => any-:up, :do => :remove_watchers
|
63
|
+
|
64
|
+
after_transition any-:up => :up, :do => :add_childs
|
65
|
+
after_transition any => [:unmonitored, :down], :do => :remove_childs
|
66
|
+
|
60
67
|
after_transition :on => :crashed, :do => :on_crashed
|
61
68
|
end
|
62
69
|
|
@@ -68,16 +75,6 @@ class Eye::Process
|
|
68
75
|
self.pid = nil
|
69
76
|
end
|
70
77
|
|
71
|
-
def on_up
|
72
|
-
add_watchers
|
73
|
-
add_childs
|
74
|
-
end
|
75
|
-
|
76
|
-
def from_up
|
77
|
-
remove_watchers
|
78
|
-
remove_childs
|
79
|
-
end
|
80
|
-
|
81
78
|
def log_transition(transition)
|
82
79
|
@states_history.push transition.to_name, @state_reason
|
83
80
|
info "switch :#{transition.event} [:#{transition.from_name} => :#{transition.to_name}] #{@state_reason ? "(reason: #{@state_reason})" : nil}"
|
@@ -40,19 +40,19 @@ class Eye::Process::StatesHistory < Eye::Utils::Tail
|
|
40
40
|
end
|
41
41
|
|
42
42
|
def any?(*seq)
|
43
|
-
states.any? do |st|
|
43
|
+
states.any? do |st|
|
44
44
|
seq.flatten.include?(st)
|
45
45
|
end
|
46
46
|
end
|
47
47
|
|
48
48
|
def noone?(*seq)
|
49
|
-
!states.all? do |st|
|
49
|
+
!states.all? do |st|
|
50
50
|
seq.flatten.include?(st)
|
51
51
|
end
|
52
52
|
end
|
53
53
|
|
54
54
|
def all?(*seq)
|
55
|
-
states.all? do |st|
|
55
|
+
states.all? do |st|
|
56
56
|
seq.flatten.include?(st)
|
57
57
|
end
|
58
58
|
end
|
data/lib/eye/process/system.rb
CHANGED
@@ -2,9 +2,9 @@ require 'timeout'
|
|
2
2
|
|
3
3
|
module Eye::Process::System
|
4
4
|
|
5
|
-
def load_pid_from_file
|
5
|
+
def load_pid_from_file
|
6
6
|
if File.exists?(self[:pid_file_ex])
|
7
|
-
_pid = File.read(self[:pid_file_ex]).to_i
|
7
|
+
_pid = File.read(self[:pid_file_ex]).to_i
|
8
8
|
_pid > 0 ? _pid : nil
|
9
9
|
end
|
10
10
|
end
|
@@ -25,9 +25,10 @@ module Eye::Process::System
|
|
25
25
|
end
|
26
26
|
|
27
27
|
def clear_pid_file
|
28
|
+
info "delete pid_file: #{self[:pid_file_ex]}"
|
28
29
|
File.unlink(self[:pid_file_ex])
|
29
30
|
true
|
30
|
-
rescue
|
31
|
+
rescue
|
31
32
|
nil
|
32
33
|
end
|
33
34
|
|
@@ -51,9 +52,18 @@ module Eye::Process::System
|
|
51
52
|
res[:result] == :ok
|
52
53
|
end
|
53
54
|
|
54
|
-
# non blocking actor timeout
|
55
55
|
def wait_for_condition(timeout, step = 0.1, &block)
|
56
|
-
|
56
|
+
res = nil
|
57
|
+
sumtime = 0
|
58
|
+
|
59
|
+
loop do
|
60
|
+
tm = Time.now
|
61
|
+
res = yield # note that yield can block actor here and timeout can be overhead
|
62
|
+
return res if res
|
63
|
+
sleep step.to_f
|
64
|
+
sumtime += (Time.now - tm)
|
65
|
+
return false if sumtime > timeout
|
66
|
+
end
|
57
67
|
end
|
58
68
|
|
59
69
|
def execute(cmd, cfg = {})
|
@@ -63,7 +73,7 @@ module Eye::Process::System
|
|
63
73
|
def failsafe_load_pid
|
64
74
|
pid = load_pid_from_file
|
65
75
|
|
66
|
-
if !pid
|
76
|
+
if !pid
|
67
77
|
# this is can be symlink changed case
|
68
78
|
sleep 0.1
|
69
79
|
pid = load_pid_from_file
|
@@ -73,25 +83,11 @@ module Eye::Process::System
|
|
73
83
|
end
|
74
84
|
|
75
85
|
def failsafe_save_pid
|
76
|
-
save_pid_to_file
|
86
|
+
save_pid_to_file
|
77
87
|
true
|
78
88
|
rescue => ex
|
79
89
|
error "failsafe_save_pid: #{ex.message}"
|
80
90
|
false
|
81
91
|
end
|
82
92
|
|
83
|
-
private
|
84
|
-
|
85
|
-
def wait_for_condition_sync(timeout, step, &block)
|
86
|
-
res = nil
|
87
|
-
|
88
|
-
Timeout::timeout(timeout.to_f) do
|
89
|
-
sleep step.to_f until res = yield
|
90
|
-
end
|
91
|
-
|
92
|
-
res
|
93
|
-
rescue Timeout::Error
|
94
|
-
false
|
95
|
-
end
|
96
|
-
|
97
93
|
end
|
data/lib/eye/process/trigger.rb
CHANGED
@@ -4,7 +4,7 @@ module Eye::Process::Trigger
|
|
4
4
|
if self[:triggers]
|
5
5
|
self[:triggers].each do |type, cfg|
|
6
6
|
add_trigger(cfg)
|
7
|
-
end
|
7
|
+
end
|
8
8
|
end
|
9
9
|
end
|
10
10
|
|
@@ -12,43 +12,24 @@ module Eye::Process::Trigger
|
|
12
12
|
self.triggers = []
|
13
13
|
end
|
14
14
|
|
15
|
-
def check_triggers
|
15
|
+
def check_triggers(transition)
|
16
16
|
return if unmonitored?
|
17
|
+
self.triggers.each { |trigger| trigger.notify(transition) }
|
18
|
+
end
|
17
19
|
|
18
|
-
|
19
|
-
|
20
|
-
|
21
|
-
|
22
|
-
|
20
|
+
def retry_start_after_flapping
|
21
|
+
return unless unmonitored?
|
22
|
+
return unless state_reason.to_s.include?('flapping') # TODO: remove hackety
|
23
|
+
|
24
|
+
schedule :start, Eye::Reason.new(:'retry start after flapping')
|
25
|
+
self.flapping_times += 1
|
23
26
|
end
|
24
27
|
|
25
28
|
private
|
26
29
|
|
27
30
|
def add_trigger(cfg = {})
|
28
|
-
trigger = Eye::Trigger.create(
|
31
|
+
trigger = Eye::Trigger.create(current_actor, cfg)
|
29
32
|
self.triggers << trigger
|
30
33
|
end
|
31
34
|
|
32
|
-
def on_flapping(trigger)
|
33
|
-
notify :error, 'flapping!'
|
34
|
-
schedule :unmonitor, Eye::Reason.new(:flapping)
|
35
|
-
|
36
|
-
@retry_times ||= 0
|
37
|
-
retry_in = trigger.retry_in
|
38
|
-
|
39
|
-
return unless retry_in
|
40
|
-
return if trigger.retry_times && @retry_times >= trigger.retry_times
|
41
|
-
|
42
|
-
schedule_in(retry_in.to_f, :retry_action)
|
43
|
-
end
|
44
|
-
|
45
|
-
def retry_action
|
46
|
-
debug "trigger retry timer"
|
47
|
-
return unless unmonitored?
|
48
|
-
return unless state_reason.to_s.include?('flapping') # TODO: remove hackety
|
49
|
-
|
50
|
-
schedule :start, Eye::Reason.new(:'retry start after flapping')
|
51
|
-
@retry_times += 1
|
52
|
-
end
|
53
|
-
|
54
35
|
end
|
data/lib/eye/process/watchers.rb
CHANGED
@@ -7,13 +7,13 @@ module Eye::Process::Watchers
|
|
7
7
|
|
8
8
|
if @watchers.blank?
|
9
9
|
# default watcher :check_alive
|
10
|
-
add_watcher(:check_alive, self[:check_alive_period]) do
|
10
|
+
add_watcher(:check_alive, self[:check_alive_period]) do
|
11
11
|
check_alive
|
12
12
|
end
|
13
13
|
|
14
14
|
# monitor childs pids
|
15
15
|
if self[:monitor_children]
|
16
|
-
add_watcher(:check_childs, self[:childs_update_period]) do
|
16
|
+
add_watcher(:check_childs, self[:childs_update_period]) do
|
17
17
|
add_or_update_childs
|
18
18
|
end
|
19
19
|
end
|
@@ -24,13 +24,13 @@ module Eye::Process::Watchers
|
|
24
24
|
warn 'try add_watchers, but its already here'
|
25
25
|
end
|
26
26
|
end
|
27
|
-
|
27
|
+
|
28
28
|
def remove_watchers
|
29
29
|
@watchers.each{|_, h| h[:timer].cancel }
|
30
30
|
@watchers = {}
|
31
31
|
end
|
32
32
|
|
33
|
-
private
|
33
|
+
private
|
34
34
|
|
35
35
|
def add_watcher(type, period = 2, subject = nil, &block)
|
36
36
|
return if @watchers[type]
|
@@ -40,17 +40,17 @@ private
|
|
40
40
|
timer = every(period.to_f) do
|
41
41
|
debug "check #{type}"
|
42
42
|
block.call(subject)
|
43
|
-
end
|
43
|
+
end
|
44
44
|
|
45
45
|
@watchers[type] ||= {:timer => timer, :subject => subject}
|
46
|
-
end
|
47
|
-
|
46
|
+
end
|
47
|
+
|
48
48
|
def start_checkers
|
49
49
|
self[:checks].each{|name, cfg| start_checker(name, cfg) }
|
50
50
|
end
|
51
51
|
|
52
52
|
def start_checker(name, cfg)
|
53
|
-
subject = Eye::Checker.create(pid, cfg,
|
53
|
+
subject = Eye::Checker.create(pid, cfg, current_actor)
|
54
54
|
|
55
55
|
# ex: {:type => :memory, :every => 5.seconds, :below => 100.megabytes, :times => [3,5]}
|
56
56
|
add_watcher("check_#{name}".to_sym, subject.every, subject, &method(:watcher_tick).to_proc)
|
@@ -59,7 +59,7 @@ private
|
|
59
59
|
def watcher_tick(subject)
|
60
60
|
unless subject.check
|
61
61
|
return unless up?
|
62
|
-
|
62
|
+
|
63
63
|
action = subject.fire || :restart
|
64
64
|
notify :warn, "Bounded #{subject.check_name}: #{subject.last_human_values} send to :#{action}"
|
65
65
|
schedule action, Eye::Reason.new("bounded #{subject.check_name}")
|
data/lib/eye/server.rb
CHANGED
@@ -3,9 +3,9 @@ require 'celluloid/autostart'
|
|
3
3
|
|
4
4
|
class Eye::Server
|
5
5
|
include Celluloid::IO
|
6
|
-
|
6
|
+
|
7
7
|
attr_reader :socket_path, :server
|
8
|
-
|
8
|
+
|
9
9
|
def initialize(socket_path)
|
10
10
|
@socket_path = socket_path
|
11
11
|
@server = begin
|
@@ -21,14 +21,22 @@ class Eye::Server
|
|
21
21
|
end
|
22
22
|
|
23
23
|
def handle_connection(socket)
|
24
|
-
|
24
|
+
text = socket.read
|
25
|
+
|
26
|
+
begin
|
27
|
+
command, *args = Marshal.load(text)
|
28
|
+
rescue => ex
|
29
|
+
error "Failed socket read #{ex.message}"
|
30
|
+
return
|
31
|
+
end
|
32
|
+
|
25
33
|
response = command(command, *args)
|
26
34
|
socket.write(Marshal.dump(response))
|
27
|
-
|
35
|
+
|
28
36
|
rescue Errno::EPIPE
|
29
37
|
# client timeouted
|
30
38
|
# do nothing
|
31
|
-
|
39
|
+
|
32
40
|
ensure
|
33
41
|
socket.close
|
34
42
|
end
|
@@ -43,7 +51,7 @@ class Eye::Server
|
|
43
51
|
end
|
44
52
|
|
45
53
|
finalizer :close_socket
|
46
|
-
|
54
|
+
|
47
55
|
def close_socket
|
48
56
|
@server.close if @server
|
49
57
|
unlink_socket_file
|
data/lib/eye/settings.rb
CHANGED
@@ -2,7 +2,7 @@ require 'fileutils'
|
|
2
2
|
|
3
3
|
module Eye::Settings
|
4
4
|
module_function
|
5
|
-
|
5
|
+
|
6
6
|
def dir
|
7
7
|
if root?
|
8
8
|
'/var/run/eye'
|
@@ -26,7 +26,7 @@ module Eye::Settings
|
|
26
26
|
def home
|
27
27
|
ENV['EYE_HOME'] || ENV['HOME']
|
28
28
|
end
|
29
|
-
|
29
|
+
|
30
30
|
def path(path)
|
31
31
|
File.join(dir, path)
|
32
32
|
end
|
@@ -38,11 +38,11 @@ module Eye::Settings
|
|
38
38
|
def socket_path
|
39
39
|
path(ENV['EYE_SOCK'] || "sock#{ENV['EYE_V']}")
|
40
40
|
end
|
41
|
-
|
41
|
+
|
42
42
|
def pid_path
|
43
43
|
path(ENV['EYE_PID'] || "pid#{ENV['EYE_V']}")
|
44
44
|
end
|
45
|
-
|
45
|
+
|
46
46
|
def cache_path
|
47
47
|
path("processes#{ENV['EYE_V']}.cache")
|
48
48
|
end
|
data/lib/eye/system.rb
CHANGED
@@ -29,7 +29,7 @@ module Eye::System
|
|
29
29
|
|
30
30
|
# Send signal to process (uses for kill)
|
31
31
|
# code: TERM(15), KILL(9), QUIT(3), ...
|
32
|
-
def send_signal(pid, code = :TERM)
|
32
|
+
def send_signal(pid, code = :TERM)
|
33
33
|
code = 0 if code == '0'
|
34
34
|
if code.to_s.to_i != 0
|
35
35
|
code = code.to_i
|
@@ -38,7 +38,7 @@ module Eye::System
|
|
38
38
|
code = code.to_s.upcase if code.is_a?(String) || code.is_a?(Symbol)
|
39
39
|
|
40
40
|
if pid
|
41
|
-
::Process.kill(code, pid)
|
41
|
+
::Process.kill(code, pid)
|
42
42
|
{:result => :ok}
|
43
43
|
else
|
44
44
|
{:error => Exception.new('no_pid')}
|
@@ -59,7 +59,7 @@ module Eye::System
|
|
59
59
|
pid = Process::spawn(prepare_env(cfg), *Shellwords.shellwords(cmd), opts)
|
60
60
|
Process.detach(pid)
|
61
61
|
{:pid => pid}
|
62
|
-
|
62
|
+
|
63
63
|
rescue Errno::ENOENT, Errno::EACCES => ex
|
64
64
|
{:error => ex}
|
65
65
|
end
|
@@ -82,7 +82,7 @@ module Eye::System
|
|
82
82
|
|
83
83
|
rescue Timeout::Error => ex
|
84
84
|
if pid
|
85
|
-
|
85
|
+
warn "[#{cfg[:name]}] send signal 9 to #{pid} (because of timeouted<#{timeout}> execution)"
|
86
86
|
send_signal(pid, 9)
|
87
87
|
end
|
88
88
|
{:error => ex}
|
@@ -90,7 +90,7 @@ module Eye::System
|
|
90
90
|
rescue Errno::ENOENT, Errno::EACCES => ex
|
91
91
|
{:error => ex}
|
92
92
|
|
93
|
-
ensure
|
93
|
+
ensure
|
94
94
|
Process.detach(pid) if pid
|
95
95
|
end
|
96
96
|
|
@@ -102,7 +102,7 @@ module Eye::System
|
|
102
102
|
h = {}
|
103
103
|
str.each_line do |line|
|
104
104
|
chunk = line.strip.split(/\s+/)
|
105
|
-
h[chunk[0].to_i] = { :ppid => chunk[1].to_i, :cpu => chunk[2].to_i,
|
105
|
+
h[chunk[0].to_i] = { :ppid => chunk[1].to_i, :cpu => chunk[2].to_i,
|
106
106
|
:rss => chunk[3].to_i, :start_time => chunk[4] }
|
107
107
|
end
|
108
108
|
h
|
@@ -114,7 +114,10 @@ module Eye::System
|
|
114
114
|
end
|
115
115
|
|
116
116
|
def host
|
117
|
-
@host ||=
|
117
|
+
@host ||= begin
|
118
|
+
require 'socket'
|
119
|
+
Socket.gethostname
|
120
|
+
end
|
118
121
|
end
|
119
122
|
|
120
123
|
# set host for tests
|