eye 0.7 → 0.8.celluloid15
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/.rubocop.yml +141 -0
- data/.travis.yml +5 -3
- data/CHANGES.md +9 -1
- data/README.md +5 -2
- data/Rakefile +6 -6
- data/bin/leye +9 -4
- data/bin/loader_eye +14 -15
- data/examples/custom_check.eye +24 -0
- data/examples/custom_trigger.eye +30 -0
- data/examples/delayed_job.eye +3 -3
- data/examples/dependency.eye +10 -11
- data/examples/leye_example/Eyefile +10 -0
- data/examples/notify.eye +3 -4
- data/examples/plugin/main.eye +5 -5
- data/examples/plugin/plugin.rb +10 -2
- data/examples/process_thin.rb +8 -8
- data/examples/processes/em.rb +18 -12
- data/examples/processes/forking.rb +5 -5
- data/examples/processes/sample.rb +46 -44
- data/examples/puma.eye +9 -8
- data/examples/rbenv.eye +5 -5
- data/examples/sidekiq.eye +3 -3
- data/examples/stress_test.eye +4 -4
- data/examples/syslog.eye +1 -1
- data/examples/test.eye +1 -2
- data/examples/thin-farm.eye +7 -8
- data/examples/triggers.eye +13 -15
- data/examples/unicorn.eye +12 -13
- data/eye.gemspec +16 -14
- data/lib/eye.rb +2 -3
- data/lib/eye/application.rb +5 -6
- data/lib/eye/checker.rb +44 -25
- data/lib/eye/checker/children_count.rb +1 -1
- data/lib/eye/checker/file_ctime.rb +1 -1
- data/lib/eye/checker/http.rb +13 -15
- data/lib/eye/checker/nop.rb +1 -0
- data/lib/eye/checker/socket.rb +60 -63
- data/lib/eye/checker/ssl_socket.rb +5 -5
- data/lib/eye/child_process.rb +6 -4
- data/lib/eye/cli.rb +74 -46
- data/lib/eye/cli/commands.rb +4 -5
- data/lib/eye/cli/render.rb +61 -41
- data/lib/eye/cli/server.rb +19 -16
- data/lib/eye/client.rb +1 -0
- data/lib/eye/config.rb +36 -33
- data/lib/eye/controller.rb +2 -3
- data/lib/eye/controller/commands.rb +1 -1
- data/lib/eye/controller/helpers.rb +2 -2
- data/lib/eye/controller/load.rb +19 -17
- data/lib/eye/controller/options.rb +1 -5
- data/lib/eye/controller/send_command.rb +21 -23
- data/lib/eye/controller/status.rb +17 -14
- data/lib/eye/dsl.rb +6 -1
- data/lib/eye/dsl/application_opts.rb +4 -3
- data/lib/eye/dsl/chain.rb +2 -2
- data/lib/eye/dsl/child_process_opts.rb +3 -3
- data/lib/eye/dsl/config_opts.rb +7 -7
- data/lib/eye/dsl/group_opts.rb +3 -3
- data/lib/eye/dsl/helpers.rb +1 -1
- data/lib/eye/dsl/main.rb +4 -3
- data/lib/eye/dsl/opts.rb +31 -28
- data/lib/eye/dsl/process_opts.rb +13 -7
- data/lib/eye/dsl/pure_opts.rb +13 -9
- data/lib/eye/dsl/validation.rb +48 -35
- data/lib/eye/group.rb +23 -8
- data/lib/eye/group/chain.rb +6 -6
- data/lib/eye/loader.rb +3 -3
- data/lib/eye/local.rb +9 -4
- data/lib/eye/logger.rb +11 -4
- data/lib/eye/notify.rb +10 -6
- data/lib/eye/notify/jabber.rb +1 -1
- data/lib/eye/notify/mail.rb +2 -2
- data/lib/eye/notify/slack.rb +4 -3
- data/lib/eye/process.rb +2 -0
- data/lib/eye/process/children.rb +4 -4
- data/lib/eye/process/commands.rb +38 -39
- data/lib/eye/process/config.rb +22 -16
- data/lib/eye/process/controller.rb +5 -19
- data/lib/eye/process/data.rb +11 -9
- data/lib/eye/process/monitor.rb +86 -76
- data/lib/eye/process/notify.rb +10 -10
- data/lib/eye/process/scheduler.rb +36 -31
- data/lib/eye/process/states.rb +7 -5
- data/lib/eye/process/states_history.rb +9 -3
- data/lib/eye/process/system.rb +35 -20
- data/lib/eye/process/trigger.rb +1 -5
- data/lib/eye/process/watchers.rb +12 -9
- data/lib/eye/reason.rb +4 -1
- data/lib/eye/server.rb +3 -2
- data/lib/eye/system.rb +22 -15
- data/lib/eye/system_resources.rb +17 -8
- data/lib/eye/trigger.rb +18 -16
- data/lib/eye/trigger/check_dependency.rb +7 -4
- data/lib/eye/trigger/flapping.rb +24 -7
- data/lib/eye/trigger/starting_guard.rb +7 -6
- data/lib/eye/trigger/stop_children.rb +2 -2
- data/lib/eye/trigger/transition.rb +1 -1
- data/lib/eye/trigger/wait_dependency.rb +3 -2
- data/lib/eye/utils.rb +4 -3
- data/lib/eye/utils/alive_array.rb +9 -4
- data/lib/eye/utils/celluloid_chain.rb +12 -10
- data/lib/eye/utils/mini_active_support.rb +16 -16
- data/lib/eye/utils/pmap.rb +2 -0
- data/lib/eye/utils/tail.rb +2 -2
- metadata +39 -8
- data/lib/eye/utils/leak_19.rb +0 -10
data/lib/eye/process/trigger.rb
CHANGED
data/lib/eye/process/watchers.rb
CHANGED
@@ -11,6 +11,12 @@ module Eye::Process::Watchers
|
|
11
11
|
check_alive
|
12
12
|
end
|
13
13
|
|
14
|
+
if self[:check_identity]
|
15
|
+
add_watcher(:check_identity, self[:check_identity_period]) do
|
16
|
+
check_identity
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
14
20
|
# monitor children pids
|
15
21
|
if self[:monitor_children]
|
16
22
|
add_watcher(:check_children, self[:children_update_period]) do
|
@@ -26,7 +32,7 @@ module Eye::Process::Watchers
|
|
26
32
|
end
|
27
33
|
|
28
34
|
def remove_watchers
|
29
|
-
@watchers.each{|_, h| h[:timer].cancel }
|
35
|
+
@watchers.each { |_, h| h[:timer].cancel }
|
30
36
|
@watchers = {}
|
31
37
|
end
|
32
38
|
|
@@ -42,25 +48,22 @@ private
|
|
42
48
|
block.call(subject)
|
43
49
|
end
|
44
50
|
|
45
|
-
@watchers[type] ||= {:
|
51
|
+
@watchers[type] ||= { timer: timer, subject: subject }
|
46
52
|
end
|
47
53
|
|
48
54
|
def start_checkers
|
49
|
-
self[:checks].each{|name, cfg| start_checker(name, cfg) }
|
55
|
+
self[:checks].each { |name, cfg| start_checker(name, cfg) }
|
50
56
|
end
|
51
57
|
|
52
58
|
def start_checker(name, cfg)
|
59
|
+
# cfg: {:type => :memory, :every => 5.seconds, :below => 100.megabytes, :times => [3, 5]}
|
53
60
|
subject = Eye::Checker.create(pid, cfg, current_actor)
|
54
|
-
|
55
|
-
# ex: {:type => :memory, :every => 5.seconds, :below => 100.megabytes, :times => [3,5]}
|
56
61
|
add_watcher("check_#{name}".to_sym, subject.every, subject, &method(:watcher_tick).to_proc) if subject
|
57
62
|
end
|
58
63
|
|
59
64
|
def watcher_tick(subject)
|
60
|
-
|
61
|
-
|
62
|
-
subject.fire
|
63
|
-
end
|
65
|
+
# double up? test needed because state can changed while subject.check
|
66
|
+
subject.fire if up? && !subject.check && up?
|
64
67
|
end
|
65
68
|
|
66
69
|
end
|
data/lib/eye/reason.rb
CHANGED
data/lib/eye/server.rb
CHANGED
data/lib/eye/system.rb
CHANGED
@@ -3,7 +3,9 @@ require 'etc'
|
|
3
3
|
require 'timeout'
|
4
4
|
|
5
5
|
module Eye::System
|
6
|
+
|
6
7
|
class << self
|
8
|
+
|
7
9
|
# Check that pid really exits
|
8
10
|
# very fast
|
9
11
|
# return result hash
|
@@ -14,17 +16,21 @@ module Eye::System
|
|
14
16
|
false
|
15
17
|
end
|
16
18
|
|
17
|
-
{:
|
19
|
+
{ result: res }
|
18
20
|
rescue => ex
|
19
|
-
{:
|
21
|
+
{ error: ex }
|
20
22
|
end
|
21
23
|
|
22
24
|
# Check that pid really exits
|
23
25
|
# very fast
|
24
26
|
# return true/false
|
25
27
|
def pid_alive?(pid)
|
26
|
-
|
27
|
-
|
28
|
+
if pid
|
29
|
+
::Process.kill(0, pid)
|
30
|
+
true
|
31
|
+
end
|
32
|
+
rescue
|
33
|
+
false
|
28
34
|
end
|
29
35
|
|
30
36
|
# Send signal to process (uses for kill)
|
@@ -39,13 +45,13 @@ module Eye::System
|
|
39
45
|
|
40
46
|
if pid
|
41
47
|
::Process.kill(code, pid)
|
42
|
-
{:
|
48
|
+
{ result: :ok }
|
43
49
|
else
|
44
|
-
{:
|
50
|
+
{ error: Exception.new('no_pid') }
|
45
51
|
end
|
46
52
|
|
47
53
|
rescue => ex
|
48
|
-
{:
|
54
|
+
{ error: ex }
|
49
55
|
end
|
50
56
|
|
51
57
|
# Daemonize cmd, and detach
|
@@ -55,12 +61,12 @@ module Eye::System
|
|
55
61
|
# :environment
|
56
62
|
# :stdin, :stdout, :stderr
|
57
63
|
def daemonize(cmd, cfg = {})
|
58
|
-
pid = ::Process
|
64
|
+
pid = ::Process.spawn(prepare_env(cfg), *Shellwords.shellwords(cmd), spawn_options(cfg))
|
59
65
|
|
60
|
-
{:
|
66
|
+
{ pid: pid, exitstatus: 0 }
|
61
67
|
|
62
68
|
rescue Errno::ENOENT, Errno::EACCES => ex
|
63
|
-
{:
|
69
|
+
{ error: ex }
|
64
70
|
|
65
71
|
ensure
|
66
72
|
Process.detach(pid) if pid
|
@@ -72,7 +78,7 @@ module Eye::System
|
|
72
78
|
# :environment
|
73
79
|
# :stdin, :stdout, :stderr
|
74
80
|
def execute(cmd, cfg = {})
|
75
|
-
pid = ::Process
|
81
|
+
pid = ::Process.spawn(prepare_env(cfg), *Shellwords.shellwords(cmd), spawn_options(cfg))
|
76
82
|
|
77
83
|
timeout = cfg[:timeout] || 1.second
|
78
84
|
status = 0
|
@@ -82,17 +88,17 @@ module Eye::System
|
|
82
88
|
status = st.exitstatus || st.termsig
|
83
89
|
end
|
84
90
|
|
85
|
-
{:
|
91
|
+
{ pid: pid, exitstatus: status }
|
86
92
|
|
87
93
|
rescue Timeout::Error => ex
|
88
94
|
if pid
|
89
95
|
warn "[#{cfg[:name]}] sending :KILL signal to <#{pid}> due to timeout (#{timeout}s)"
|
90
96
|
send_signal(pid, 9)
|
91
97
|
end
|
92
|
-
{:
|
98
|
+
{ error: ex }
|
93
99
|
|
94
100
|
rescue Errno::ENOENT, Errno::EACCES => ex
|
95
|
-
{:
|
101
|
+
{ error: ex }
|
96
102
|
|
97
103
|
ensure
|
98
104
|
Process.detach(pid) if pid
|
@@ -127,12 +133,13 @@ module Eye::System
|
|
127
133
|
def prepare_env(config = {})
|
128
134
|
env = {}
|
129
135
|
|
130
|
-
(config[:environment] || {}).each do |k,v|
|
136
|
+
(config[:environment] || {}).each do |k, v|
|
131
137
|
env[k.to_s] = v && v.to_s
|
132
138
|
end
|
133
139
|
|
134
140
|
env
|
135
141
|
end
|
142
|
+
|
136
143
|
end
|
137
144
|
|
138
145
|
end
|
data/lib/eye/system_resources.rb
CHANGED
@@ -45,7 +45,7 @@ class Eye::SystemResources
|
|
45
45
|
end
|
46
46
|
|
47
47
|
def deep_children(pid)
|
48
|
-
Array(pid_or_children(pid)).flatten.sort_by
|
48
|
+
Array(pid_or_children(pid)).flatten.sort_by(&:-@)
|
49
49
|
end
|
50
50
|
|
51
51
|
def pid_or_children(pid)
|
@@ -57,20 +57,25 @@ class Eye::SystemResources
|
|
57
57
|
end
|
58
58
|
end
|
59
59
|
|
60
|
+
def args(pid)
|
61
|
+
Eye::Sigar.proc_args(pid).join(' ').strip rescue '-'
|
62
|
+
end
|
63
|
+
|
60
64
|
def resources(pid)
|
61
|
-
{ :
|
62
|
-
:
|
63
|
-
:
|
64
|
-
:pid
|
65
|
-
}
|
65
|
+
{ memory: memory(pid),
|
66
|
+
cpu: cpu(pid),
|
67
|
+
start_time: start_time(pid),
|
68
|
+
pid: pid }
|
66
69
|
end
|
67
70
|
|
68
71
|
def cache
|
69
72
|
Celluloid::Actor[:system_resources_cache]
|
70
73
|
end
|
74
|
+
|
71
75
|
end
|
72
76
|
|
73
77
|
class Cache
|
78
|
+
|
74
79
|
include Celluloid
|
75
80
|
|
76
81
|
attr_reader :expire
|
@@ -95,13 +100,15 @@ class Eye::SystemResources
|
|
95
100
|
def proc_mem(pid)
|
96
101
|
@memory[pid] ||= Eye::Sigar.proc_mem(pid) if pid
|
97
102
|
|
98
|
-
rescue ArgumentError
|
103
|
+
rescue ArgumentError
|
104
|
+
# when incorrect PID, just skip
|
99
105
|
end
|
100
106
|
|
101
107
|
def proc_cpu(pid)
|
102
108
|
@cpu[pid] ||= Eye::Sigar.proc_cpu(pid) if pid
|
103
109
|
|
104
|
-
rescue ArgumentError
|
110
|
+
rescue ArgumentError
|
111
|
+
# when incorrect PID, just skip
|
105
112
|
end
|
106
113
|
|
107
114
|
def children(pid)
|
@@ -111,8 +118,10 @@ class Eye::SystemResources
|
|
111
118
|
[]
|
112
119
|
end
|
113
120
|
end
|
121
|
+
|
114
122
|
end
|
115
123
|
|
116
124
|
# Setup global sigar singleton here
|
117
125
|
Cache.supervise_as(:system_resources_cache)
|
126
|
+
|
118
127
|
end
|
data/lib/eye/trigger.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
class Eye::Trigger
|
2
|
+
|
2
3
|
include Eye::Dsl::Validation
|
3
4
|
|
4
5
|
autoload :Flapping, 'eye/trigger/flapping'
|
@@ -8,19 +9,18 @@ class Eye::Trigger
|
|
8
9
|
autoload :CheckDependency, 'eye/trigger/check_dependency'
|
9
10
|
autoload :StartingGuard, 'eye/trigger/starting_guard'
|
10
11
|
|
11
|
-
TYPES = {:
|
12
|
-
|
13
|
-
}
|
12
|
+
TYPES = { flapping: 'Flapping', transition: 'Transition', stop_children: 'StopChildren',
|
13
|
+
wait_dependency: 'WaitDependency', check_dependency: 'CheckDependency', starting_guard: 'StartingGuard' }
|
14
14
|
|
15
15
|
attr_reader :message, :options, :process
|
16
16
|
|
17
17
|
def self.name_and_class(type)
|
18
18
|
type = type.to_sym
|
19
|
-
return {:
|
19
|
+
return { name: type, type: type } if TYPES[type]
|
20
20
|
|
21
|
-
if type =~
|
22
|
-
ctype =
|
23
|
-
return {:
|
21
|
+
if type =~ %r[\A(.*?)_?[0-9]+\z]
|
22
|
+
ctype = Regexp.last_match(1).to_sym
|
23
|
+
return { name: type, type: ctype } if TYPES[ctype]
|
24
24
|
end
|
25
25
|
end
|
26
26
|
|
@@ -36,7 +36,7 @@ class Eye::Trigger
|
|
36
36
|
def self.create(process, options = {})
|
37
37
|
get_class(options[:type]).new(process, options)
|
38
38
|
|
39
|
-
rescue
|
39
|
+
rescue Object => ex
|
40
40
|
log_ex(ex)
|
41
41
|
nil
|
42
42
|
end
|
@@ -72,7 +72,7 @@ class Eye::Trigger
|
|
72
72
|
|
73
73
|
check(transition) if filter_transition(transition)
|
74
74
|
|
75
|
-
rescue
|
75
|
+
rescue Object => ex
|
76
76
|
if ex.class == Eye::Process::StateError
|
77
77
|
raise ex
|
78
78
|
else
|
@@ -92,7 +92,7 @@ class Eye::Trigger
|
|
92
92
|
compare_state(trans.event, event)
|
93
93
|
end
|
94
94
|
|
95
|
-
def check(
|
95
|
+
def check(_transition)
|
96
96
|
raise NotImplementedError
|
97
97
|
end
|
98
98
|
|
@@ -126,22 +126,24 @@ class Eye::Trigger
|
|
126
126
|
end
|
127
127
|
|
128
128
|
class Custom < Eye::Trigger
|
129
|
+
|
129
130
|
def self.inherited(base)
|
130
131
|
super
|
131
132
|
register(base)
|
132
133
|
end
|
134
|
+
|
133
135
|
end
|
134
136
|
|
135
137
|
private
|
136
138
|
|
137
139
|
def compare_state(state_name, condition)
|
138
140
|
case condition
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
141
|
+
when Symbol
|
142
|
+
state_name == condition
|
143
|
+
when Array
|
144
|
+
condition.include?(state_name)
|
145
|
+
else
|
146
|
+
true
|
145
147
|
end
|
146
148
|
end
|
147
149
|
|
@@ -1,4 +1,5 @@
|
|
1
1
|
class Eye::Trigger::CheckDependency < Eye::Trigger
|
2
|
+
|
2
3
|
param :names, [Array], true, 5
|
3
4
|
|
4
5
|
def check(transition)
|
@@ -10,14 +11,16 @@ private
|
|
10
11
|
def check_dependency(to)
|
11
12
|
processes = names.map do |name|
|
12
13
|
Eye::Control.find_nearest_process(name, process.group_name_pure, process.app_name)
|
13
|
-
end
|
14
|
+
end
|
15
|
+
|
16
|
+
processes = processes.compact.select { |p| p.state_name != :unmonitored }
|
14
17
|
return if processes.empty?
|
15
18
|
processes = Eye::Utils::AliveArray.new(processes)
|
16
19
|
|
17
20
|
act = case to
|
18
|
-
when :down, :restarting
|
19
|
-
when :stopping
|
20
|
-
when :unmonitored
|
21
|
+
when :down, :restarting then :restart
|
22
|
+
when :stopping then :stop
|
23
|
+
when :unmonitored then :unmonitor
|
21
24
|
end
|
22
25
|
|
23
26
|
if act
|
data/lib/eye/trigger/flapping.rb
CHANGED
@@ -7,10 +7,12 @@ class Eye::Trigger::Flapping < Eye::Trigger
|
|
7
7
|
param :within, [Float, Fixnum], true
|
8
8
|
param :retry_in, [Float, Fixnum]
|
9
9
|
param :retry_times, [Fixnum]
|
10
|
+
param :reretry_in, [Float, Fixnum]
|
11
|
+
param :reretry_times, [Fixnum]
|
10
12
|
|
11
13
|
def initialize(*args)
|
12
14
|
super
|
13
|
-
|
15
|
+
clear_counters
|
14
16
|
end
|
15
17
|
|
16
18
|
def check(transition)
|
@@ -19,9 +21,16 @@ class Eye::Trigger::Flapping < Eye::Trigger
|
|
19
21
|
|
20
22
|
private
|
21
23
|
|
24
|
+
def clear_counters
|
25
|
+
@retry_times = 0
|
26
|
+
@reretry_times = 0
|
27
|
+
end
|
28
|
+
|
22
29
|
def good?
|
23
|
-
|
24
|
-
|
30
|
+
down_count = 0
|
31
|
+
process.states_history.states_for_period(within, @last_at) do |s|
|
32
|
+
down_count += 1 if s[:state] == :down
|
33
|
+
end
|
25
34
|
|
26
35
|
if down_count >= times
|
27
36
|
@last_at = process.states_history.last_state_changed_at
|
@@ -38,10 +47,18 @@ private
|
|
38
47
|
process.schedule :unmonitor, Eye::Reason::Flapping.new(:flapping)
|
39
48
|
|
40
49
|
return unless retry_in
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
50
|
+
if !retry_times || (retry_times && @retry_times < retry_times)
|
51
|
+
@retry_times += 1
|
52
|
+
process.schedule_in(retry_in.to_f, :conditional_start, Eye::Reason::Flapping.new('retry start after flapping'))
|
53
|
+
else
|
54
|
+
if reretry_in
|
55
|
+
if !reretry_times || (reretry_times && @reretry_times < reretry_times)
|
56
|
+
@retry_times = 0
|
57
|
+
@reretry_times += 1
|
58
|
+
process.schedule_in(reretry_in.to_f, :conditional_start, Eye::Reason::Flapping.new('reretry start after flapping'))
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
45
62
|
end
|
46
63
|
|
47
64
|
end
|