eye 0.7 → 0.8.celluloid15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/.rubocop.yml +141 -0
- data/.travis.yml +5 -3
- data/CHANGES.md +9 -1
- data/README.md +5 -2
- data/Rakefile +6 -6
- data/bin/leye +9 -4
- data/bin/loader_eye +14 -15
- data/examples/custom_check.eye +24 -0
- data/examples/custom_trigger.eye +30 -0
- data/examples/delayed_job.eye +3 -3
- data/examples/dependency.eye +10 -11
- data/examples/leye_example/Eyefile +10 -0
- data/examples/notify.eye +3 -4
- data/examples/plugin/main.eye +5 -5
- data/examples/plugin/plugin.rb +10 -2
- data/examples/process_thin.rb +8 -8
- data/examples/processes/em.rb +18 -12
- data/examples/processes/forking.rb +5 -5
- data/examples/processes/sample.rb +46 -44
- data/examples/puma.eye +9 -8
- data/examples/rbenv.eye +5 -5
- data/examples/sidekiq.eye +3 -3
- data/examples/stress_test.eye +4 -4
- data/examples/syslog.eye +1 -1
- data/examples/test.eye +1 -2
- data/examples/thin-farm.eye +7 -8
- data/examples/triggers.eye +13 -15
- data/examples/unicorn.eye +12 -13
- data/eye.gemspec +16 -14
- data/lib/eye.rb +2 -3
- data/lib/eye/application.rb +5 -6
- data/lib/eye/checker.rb +44 -25
- data/lib/eye/checker/children_count.rb +1 -1
- data/lib/eye/checker/file_ctime.rb +1 -1
- data/lib/eye/checker/http.rb +13 -15
- data/lib/eye/checker/nop.rb +1 -0
- data/lib/eye/checker/socket.rb +60 -63
- data/lib/eye/checker/ssl_socket.rb +5 -5
- data/lib/eye/child_process.rb +6 -4
- data/lib/eye/cli.rb +74 -46
- data/lib/eye/cli/commands.rb +4 -5
- data/lib/eye/cli/render.rb +61 -41
- data/lib/eye/cli/server.rb +19 -16
- data/lib/eye/client.rb +1 -0
- data/lib/eye/config.rb +36 -33
- data/lib/eye/controller.rb +2 -3
- data/lib/eye/controller/commands.rb +1 -1
- data/lib/eye/controller/helpers.rb +2 -2
- data/lib/eye/controller/load.rb +19 -17
- data/lib/eye/controller/options.rb +1 -5
- data/lib/eye/controller/send_command.rb +21 -23
- data/lib/eye/controller/status.rb +17 -14
- data/lib/eye/dsl.rb +6 -1
- data/lib/eye/dsl/application_opts.rb +4 -3
- data/lib/eye/dsl/chain.rb +2 -2
- data/lib/eye/dsl/child_process_opts.rb +3 -3
- data/lib/eye/dsl/config_opts.rb +7 -7
- data/lib/eye/dsl/group_opts.rb +3 -3
- data/lib/eye/dsl/helpers.rb +1 -1
- data/lib/eye/dsl/main.rb +4 -3
- data/lib/eye/dsl/opts.rb +31 -28
- data/lib/eye/dsl/process_opts.rb +13 -7
- data/lib/eye/dsl/pure_opts.rb +13 -9
- data/lib/eye/dsl/validation.rb +48 -35
- data/lib/eye/group.rb +23 -8
- data/lib/eye/group/chain.rb +6 -6
- data/lib/eye/loader.rb +3 -3
- data/lib/eye/local.rb +9 -4
- data/lib/eye/logger.rb +11 -4
- data/lib/eye/notify.rb +10 -6
- data/lib/eye/notify/jabber.rb +1 -1
- data/lib/eye/notify/mail.rb +2 -2
- data/lib/eye/notify/slack.rb +4 -3
- data/lib/eye/process.rb +2 -0
- data/lib/eye/process/children.rb +4 -4
- data/lib/eye/process/commands.rb +38 -39
- data/lib/eye/process/config.rb +22 -16
- data/lib/eye/process/controller.rb +5 -19
- data/lib/eye/process/data.rb +11 -9
- data/lib/eye/process/monitor.rb +86 -76
- data/lib/eye/process/notify.rb +10 -10
- data/lib/eye/process/scheduler.rb +36 -31
- data/lib/eye/process/states.rb +7 -5
- data/lib/eye/process/states_history.rb +9 -3
- data/lib/eye/process/system.rb +35 -20
- data/lib/eye/process/trigger.rb +1 -5
- data/lib/eye/process/watchers.rb +12 -9
- data/lib/eye/reason.rb +4 -1
- data/lib/eye/server.rb +3 -2
- data/lib/eye/system.rb +22 -15
- data/lib/eye/system_resources.rb +17 -8
- data/lib/eye/trigger.rb +18 -16
- data/lib/eye/trigger/check_dependency.rb +7 -4
- data/lib/eye/trigger/flapping.rb +24 -7
- data/lib/eye/trigger/starting_guard.rb +7 -6
- data/lib/eye/trigger/stop_children.rb +2 -2
- data/lib/eye/trigger/transition.rb +1 -1
- data/lib/eye/trigger/wait_dependency.rb +3 -2
- data/lib/eye/utils.rb +4 -3
- data/lib/eye/utils/alive_array.rb +9 -4
- data/lib/eye/utils/celluloid_chain.rb +12 -10
- data/lib/eye/utils/mini_active_support.rb +16 -16
- data/lib/eye/utils/pmap.rb +2 -0
- data/lib/eye/utils/tail.rb +2 -2
- metadata +39 -8
- data/lib/eye/utils/leak_19.rb +0 -10
data/lib/eye/process/trigger.rb
CHANGED
data/lib/eye/process/watchers.rb
CHANGED
@@ -11,6 +11,12 @@ module Eye::Process::Watchers
|
|
11
11
|
check_alive
|
12
12
|
end
|
13
13
|
|
14
|
+
if self[:check_identity]
|
15
|
+
add_watcher(:check_identity, self[:check_identity_period]) do
|
16
|
+
check_identity
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
14
20
|
# monitor children pids
|
15
21
|
if self[:monitor_children]
|
16
22
|
add_watcher(:check_children, self[:children_update_period]) do
|
@@ -26,7 +32,7 @@ module Eye::Process::Watchers
|
|
26
32
|
end
|
27
33
|
|
28
34
|
def remove_watchers
|
29
|
-
@watchers.each{|_, h| h[:timer].cancel }
|
35
|
+
@watchers.each { |_, h| h[:timer].cancel }
|
30
36
|
@watchers = {}
|
31
37
|
end
|
32
38
|
|
@@ -42,25 +48,22 @@ private
|
|
42
48
|
block.call(subject)
|
43
49
|
end
|
44
50
|
|
45
|
-
@watchers[type] ||= {:
|
51
|
+
@watchers[type] ||= { timer: timer, subject: subject }
|
46
52
|
end
|
47
53
|
|
48
54
|
def start_checkers
|
49
|
-
self[:checks].each{|name, cfg| start_checker(name, cfg) }
|
55
|
+
self[:checks].each { |name, cfg| start_checker(name, cfg) }
|
50
56
|
end
|
51
57
|
|
52
58
|
def start_checker(name, cfg)
|
59
|
+
# cfg: {:type => :memory, :every => 5.seconds, :below => 100.megabytes, :times => [3, 5]}
|
53
60
|
subject = Eye::Checker.create(pid, cfg, current_actor)
|
54
|
-
|
55
|
-
# ex: {:type => :memory, :every => 5.seconds, :below => 100.megabytes, :times => [3,5]}
|
56
61
|
add_watcher("check_#{name}".to_sym, subject.every, subject, &method(:watcher_tick).to_proc) if subject
|
57
62
|
end
|
58
63
|
|
59
64
|
def watcher_tick(subject)
|
60
|
-
|
61
|
-
|
62
|
-
subject.fire
|
63
|
-
end
|
65
|
+
# double up? test needed because state can changed while subject.check
|
66
|
+
subject.fire if up? && !subject.check && up?
|
64
67
|
end
|
65
68
|
|
66
69
|
end
|
data/lib/eye/reason.rb
CHANGED
data/lib/eye/server.rb
CHANGED
data/lib/eye/system.rb
CHANGED
@@ -3,7 +3,9 @@ require 'etc'
|
|
3
3
|
require 'timeout'
|
4
4
|
|
5
5
|
module Eye::System
|
6
|
+
|
6
7
|
class << self
|
8
|
+
|
7
9
|
# Check that pid really exits
|
8
10
|
# very fast
|
9
11
|
# return result hash
|
@@ -14,17 +16,21 @@ module Eye::System
|
|
14
16
|
false
|
15
17
|
end
|
16
18
|
|
17
|
-
{:
|
19
|
+
{ result: res }
|
18
20
|
rescue => ex
|
19
|
-
{:
|
21
|
+
{ error: ex }
|
20
22
|
end
|
21
23
|
|
22
24
|
# Check that pid really exits
|
23
25
|
# very fast
|
24
26
|
# return true/false
|
25
27
|
def pid_alive?(pid)
|
26
|
-
|
27
|
-
|
28
|
+
if pid
|
29
|
+
::Process.kill(0, pid)
|
30
|
+
true
|
31
|
+
end
|
32
|
+
rescue
|
33
|
+
false
|
28
34
|
end
|
29
35
|
|
30
36
|
# Send signal to process (uses for kill)
|
@@ -39,13 +45,13 @@ module Eye::System
|
|
39
45
|
|
40
46
|
if pid
|
41
47
|
::Process.kill(code, pid)
|
42
|
-
{:
|
48
|
+
{ result: :ok }
|
43
49
|
else
|
44
|
-
{:
|
50
|
+
{ error: Exception.new('no_pid') }
|
45
51
|
end
|
46
52
|
|
47
53
|
rescue => ex
|
48
|
-
{:
|
54
|
+
{ error: ex }
|
49
55
|
end
|
50
56
|
|
51
57
|
# Daemonize cmd, and detach
|
@@ -55,12 +61,12 @@ module Eye::System
|
|
55
61
|
# :environment
|
56
62
|
# :stdin, :stdout, :stderr
|
57
63
|
def daemonize(cmd, cfg = {})
|
58
|
-
pid = ::Process
|
64
|
+
pid = ::Process.spawn(prepare_env(cfg), *Shellwords.shellwords(cmd), spawn_options(cfg))
|
59
65
|
|
60
|
-
{:
|
66
|
+
{ pid: pid, exitstatus: 0 }
|
61
67
|
|
62
68
|
rescue Errno::ENOENT, Errno::EACCES => ex
|
63
|
-
{:
|
69
|
+
{ error: ex }
|
64
70
|
|
65
71
|
ensure
|
66
72
|
Process.detach(pid) if pid
|
@@ -72,7 +78,7 @@ module Eye::System
|
|
72
78
|
# :environment
|
73
79
|
# :stdin, :stdout, :stderr
|
74
80
|
def execute(cmd, cfg = {})
|
75
|
-
pid = ::Process
|
81
|
+
pid = ::Process.spawn(prepare_env(cfg), *Shellwords.shellwords(cmd), spawn_options(cfg))
|
76
82
|
|
77
83
|
timeout = cfg[:timeout] || 1.second
|
78
84
|
status = 0
|
@@ -82,17 +88,17 @@ module Eye::System
|
|
82
88
|
status = st.exitstatus || st.termsig
|
83
89
|
end
|
84
90
|
|
85
|
-
{:
|
91
|
+
{ pid: pid, exitstatus: status }
|
86
92
|
|
87
93
|
rescue Timeout::Error => ex
|
88
94
|
if pid
|
89
95
|
warn "[#{cfg[:name]}] sending :KILL signal to <#{pid}> due to timeout (#{timeout}s)"
|
90
96
|
send_signal(pid, 9)
|
91
97
|
end
|
92
|
-
{:
|
98
|
+
{ error: ex }
|
93
99
|
|
94
100
|
rescue Errno::ENOENT, Errno::EACCES => ex
|
95
|
-
{:
|
101
|
+
{ error: ex }
|
96
102
|
|
97
103
|
ensure
|
98
104
|
Process.detach(pid) if pid
|
@@ -127,12 +133,13 @@ module Eye::System
|
|
127
133
|
def prepare_env(config = {})
|
128
134
|
env = {}
|
129
135
|
|
130
|
-
(config[:environment] || {}).each do |k,v|
|
136
|
+
(config[:environment] || {}).each do |k, v|
|
131
137
|
env[k.to_s] = v && v.to_s
|
132
138
|
end
|
133
139
|
|
134
140
|
env
|
135
141
|
end
|
142
|
+
|
136
143
|
end
|
137
144
|
|
138
145
|
end
|
data/lib/eye/system_resources.rb
CHANGED
@@ -45,7 +45,7 @@ class Eye::SystemResources
|
|
45
45
|
end
|
46
46
|
|
47
47
|
def deep_children(pid)
|
48
|
-
Array(pid_or_children(pid)).flatten.sort_by
|
48
|
+
Array(pid_or_children(pid)).flatten.sort_by(&:-@)
|
49
49
|
end
|
50
50
|
|
51
51
|
def pid_or_children(pid)
|
@@ -57,20 +57,25 @@ class Eye::SystemResources
|
|
57
57
|
end
|
58
58
|
end
|
59
59
|
|
60
|
+
def args(pid)
|
61
|
+
Eye::Sigar.proc_args(pid).join(' ').strip rescue '-'
|
62
|
+
end
|
63
|
+
|
60
64
|
def resources(pid)
|
61
|
-
{ :
|
62
|
-
:
|
63
|
-
:
|
64
|
-
:pid
|
65
|
-
}
|
65
|
+
{ memory: memory(pid),
|
66
|
+
cpu: cpu(pid),
|
67
|
+
start_time: start_time(pid),
|
68
|
+
pid: pid }
|
66
69
|
end
|
67
70
|
|
68
71
|
def cache
|
69
72
|
Celluloid::Actor[:system_resources_cache]
|
70
73
|
end
|
74
|
+
|
71
75
|
end
|
72
76
|
|
73
77
|
class Cache
|
78
|
+
|
74
79
|
include Celluloid
|
75
80
|
|
76
81
|
attr_reader :expire
|
@@ -95,13 +100,15 @@ class Eye::SystemResources
|
|
95
100
|
def proc_mem(pid)
|
96
101
|
@memory[pid] ||= Eye::Sigar.proc_mem(pid) if pid
|
97
102
|
|
98
|
-
rescue ArgumentError
|
103
|
+
rescue ArgumentError
|
104
|
+
# when incorrect PID, just skip
|
99
105
|
end
|
100
106
|
|
101
107
|
def proc_cpu(pid)
|
102
108
|
@cpu[pid] ||= Eye::Sigar.proc_cpu(pid) if pid
|
103
109
|
|
104
|
-
rescue ArgumentError
|
110
|
+
rescue ArgumentError
|
111
|
+
# when incorrect PID, just skip
|
105
112
|
end
|
106
113
|
|
107
114
|
def children(pid)
|
@@ -111,8 +118,10 @@ class Eye::SystemResources
|
|
111
118
|
[]
|
112
119
|
end
|
113
120
|
end
|
121
|
+
|
114
122
|
end
|
115
123
|
|
116
124
|
# Setup global sigar singleton here
|
117
125
|
Cache.supervise_as(:system_resources_cache)
|
126
|
+
|
118
127
|
end
|
data/lib/eye/trigger.rb
CHANGED
@@ -1,4 +1,5 @@
|
|
1
1
|
class Eye::Trigger
|
2
|
+
|
2
3
|
include Eye::Dsl::Validation
|
3
4
|
|
4
5
|
autoload :Flapping, 'eye/trigger/flapping'
|
@@ -8,19 +9,18 @@ class Eye::Trigger
|
|
8
9
|
autoload :CheckDependency, 'eye/trigger/check_dependency'
|
9
10
|
autoload :StartingGuard, 'eye/trigger/starting_guard'
|
10
11
|
|
11
|
-
TYPES = {:
|
12
|
-
|
13
|
-
}
|
12
|
+
TYPES = { flapping: 'Flapping', transition: 'Transition', stop_children: 'StopChildren',
|
13
|
+
wait_dependency: 'WaitDependency', check_dependency: 'CheckDependency', starting_guard: 'StartingGuard' }
|
14
14
|
|
15
15
|
attr_reader :message, :options, :process
|
16
16
|
|
17
17
|
def self.name_and_class(type)
|
18
18
|
type = type.to_sym
|
19
|
-
return {:
|
19
|
+
return { name: type, type: type } if TYPES[type]
|
20
20
|
|
21
|
-
if type =~
|
22
|
-
ctype =
|
23
|
-
return {:
|
21
|
+
if type =~ %r[\A(.*?)_?[0-9]+\z]
|
22
|
+
ctype = Regexp.last_match(1).to_sym
|
23
|
+
return { name: type, type: ctype } if TYPES[ctype]
|
24
24
|
end
|
25
25
|
end
|
26
26
|
|
@@ -36,7 +36,7 @@ class Eye::Trigger
|
|
36
36
|
def self.create(process, options = {})
|
37
37
|
get_class(options[:type]).new(process, options)
|
38
38
|
|
39
|
-
rescue
|
39
|
+
rescue Object => ex
|
40
40
|
log_ex(ex)
|
41
41
|
nil
|
42
42
|
end
|
@@ -72,7 +72,7 @@ class Eye::Trigger
|
|
72
72
|
|
73
73
|
check(transition) if filter_transition(transition)
|
74
74
|
|
75
|
-
rescue
|
75
|
+
rescue Object => ex
|
76
76
|
if ex.class == Eye::Process::StateError
|
77
77
|
raise ex
|
78
78
|
else
|
@@ -92,7 +92,7 @@ class Eye::Trigger
|
|
92
92
|
compare_state(trans.event, event)
|
93
93
|
end
|
94
94
|
|
95
|
-
def check(
|
95
|
+
def check(_transition)
|
96
96
|
raise NotImplementedError
|
97
97
|
end
|
98
98
|
|
@@ -126,22 +126,24 @@ class Eye::Trigger
|
|
126
126
|
end
|
127
127
|
|
128
128
|
class Custom < Eye::Trigger
|
129
|
+
|
129
130
|
def self.inherited(base)
|
130
131
|
super
|
131
132
|
register(base)
|
132
133
|
end
|
134
|
+
|
133
135
|
end
|
134
136
|
|
135
137
|
private
|
136
138
|
|
137
139
|
def compare_state(state_name, condition)
|
138
140
|
case condition
|
139
|
-
|
140
|
-
|
141
|
-
|
142
|
-
|
143
|
-
|
144
|
-
|
141
|
+
when Symbol
|
142
|
+
state_name == condition
|
143
|
+
when Array
|
144
|
+
condition.include?(state_name)
|
145
|
+
else
|
146
|
+
true
|
145
147
|
end
|
146
148
|
end
|
147
149
|
|
@@ -1,4 +1,5 @@
|
|
1
1
|
class Eye::Trigger::CheckDependency < Eye::Trigger
|
2
|
+
|
2
3
|
param :names, [Array], true, 5
|
3
4
|
|
4
5
|
def check(transition)
|
@@ -10,14 +11,16 @@ private
|
|
10
11
|
def check_dependency(to)
|
11
12
|
processes = names.map do |name|
|
12
13
|
Eye::Control.find_nearest_process(name, process.group_name_pure, process.app_name)
|
13
|
-
end
|
14
|
+
end
|
15
|
+
|
16
|
+
processes = processes.compact.select { |p| p.state_name != :unmonitored }
|
14
17
|
return if processes.empty?
|
15
18
|
processes = Eye::Utils::AliveArray.new(processes)
|
16
19
|
|
17
20
|
act = case to
|
18
|
-
when :down, :restarting
|
19
|
-
when :stopping
|
20
|
-
when :unmonitored
|
21
|
+
when :down, :restarting then :restart
|
22
|
+
when :stopping then :stop
|
23
|
+
when :unmonitored then :unmonitor
|
21
24
|
end
|
22
25
|
|
23
26
|
if act
|
data/lib/eye/trigger/flapping.rb
CHANGED
@@ -7,10 +7,12 @@ class Eye::Trigger::Flapping < Eye::Trigger
|
|
7
7
|
param :within, [Float, Fixnum], true
|
8
8
|
param :retry_in, [Float, Fixnum]
|
9
9
|
param :retry_times, [Fixnum]
|
10
|
+
param :reretry_in, [Float, Fixnum]
|
11
|
+
param :reretry_times, [Fixnum]
|
10
12
|
|
11
13
|
def initialize(*args)
|
12
14
|
super
|
13
|
-
|
15
|
+
clear_counters
|
14
16
|
end
|
15
17
|
|
16
18
|
def check(transition)
|
@@ -19,9 +21,16 @@ class Eye::Trigger::Flapping < Eye::Trigger
|
|
19
21
|
|
20
22
|
private
|
21
23
|
|
24
|
+
def clear_counters
|
25
|
+
@retry_times = 0
|
26
|
+
@reretry_times = 0
|
27
|
+
end
|
28
|
+
|
22
29
|
def good?
|
23
|
-
|
24
|
-
|
30
|
+
down_count = 0
|
31
|
+
process.states_history.states_for_period(within, @last_at) do |s|
|
32
|
+
down_count += 1 if s[:state] == :down
|
33
|
+
end
|
25
34
|
|
26
35
|
if down_count >= times
|
27
36
|
@last_at = process.states_history.last_state_changed_at
|
@@ -38,10 +47,18 @@ private
|
|
38
47
|
process.schedule :unmonitor, Eye::Reason::Flapping.new(:flapping)
|
39
48
|
|
40
49
|
return unless retry_in
|
41
|
-
|
42
|
-
|
43
|
-
|
44
|
-
|
50
|
+
if !retry_times || (retry_times && @retry_times < retry_times)
|
51
|
+
@retry_times += 1
|
52
|
+
process.schedule_in(retry_in.to_f, :conditional_start, Eye::Reason::Flapping.new('retry start after flapping'))
|
53
|
+
else
|
54
|
+
if reretry_in
|
55
|
+
if !reretry_times || (reretry_times && @reretry_times < reretry_times)
|
56
|
+
@retry_times = 0
|
57
|
+
@reretry_times += 1
|
58
|
+
process.schedule_in(reretry_in.to_f, :conditional_start, Eye::Reason::Flapping.new('reretry start after flapping'))
|
59
|
+
end
|
60
|
+
end
|
61
|
+
end
|
45
62
|
end
|
46
63
|
|
47
64
|
end
|