ace-eye 0.6.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +7 -0
- data/.gitignore +38 -0
- data/.rspec +2 -0
- data/.travis.yml +6 -0
- data/CHANGES.md +77 -0
- data/Gemfile +6 -0
- data/LICENSE +22 -0
- data/README.md +212 -0
- data/Rakefile +35 -0
- data/bin/eye +5 -0
- data/bin/loader_eye +72 -0
- data/bin/runner +16 -0
- data/examples/dependency.eye +17 -0
- data/examples/notify.eye +19 -0
- data/examples/plugin/README.md +15 -0
- data/examples/plugin/main.eye +15 -0
- data/examples/plugin/plugin.rb +63 -0
- data/examples/process_thin.rb +29 -0
- data/examples/processes/em.rb +57 -0
- data/examples/processes/forking.rb +20 -0
- data/examples/processes/sample.rb +144 -0
- data/examples/processes/thin.ru +12 -0
- data/examples/puma.eye +29 -0
- data/examples/rbenv.eye +11 -0
- data/examples/sidekiq.eye +23 -0
- data/examples/test.eye +87 -0
- data/examples/thin-farm.eye +30 -0
- data/examples/unicorn.eye +39 -0
- data/eye.gemspec +40 -0
- data/lib/eye.rb +28 -0
- data/lib/eye/application.rb +73 -0
- data/lib/eye/checker.rb +258 -0
- data/lib/eye/checker/children_count.rb +44 -0
- data/lib/eye/checker/children_memory.rb +12 -0
- data/lib/eye/checker/cpu.rb +17 -0
- data/lib/eye/checker/cputime.rb +13 -0
- data/lib/eye/checker/file_ctime.rb +24 -0
- data/lib/eye/checker/file_size.rb +34 -0
- data/lib/eye/checker/file_touched.rb +15 -0
- data/lib/eye/checker/http.rb +96 -0
- data/lib/eye/checker/memory.rb +17 -0
- data/lib/eye/checker/nop.rb +6 -0
- data/lib/eye/checker/runtime.rb +18 -0
- data/lib/eye/checker/socket.rb +159 -0
- data/lib/eye/child_process.rb +101 -0
- data/lib/eye/cli.rb +185 -0
- data/lib/eye/cli/commands.rb +78 -0
- data/lib/eye/cli/render.rb +130 -0
- data/lib/eye/cli/server.rb +93 -0
- data/lib/eye/client.rb +32 -0
- data/lib/eye/config.rb +91 -0
- data/lib/eye/control.rb +2 -0
- data/lib/eye/controller.rb +54 -0
- data/lib/eye/controller/commands.rb +88 -0
- data/lib/eye/controller/helpers.rb +101 -0
- data/lib/eye/controller/load.rb +224 -0
- data/lib/eye/controller/options.rb +18 -0
- data/lib/eye/controller/send_command.rb +177 -0
- data/lib/eye/controller/status.rb +72 -0
- data/lib/eye/dsl.rb +53 -0
- data/lib/eye/dsl/application_opts.rb +39 -0
- data/lib/eye/dsl/chain.rb +12 -0
- data/lib/eye/dsl/child_process_opts.rb +13 -0
- data/lib/eye/dsl/config_opts.rb +55 -0
- data/lib/eye/dsl/group_opts.rb +32 -0
- data/lib/eye/dsl/helpers.rb +20 -0
- data/lib/eye/dsl/main.rb +51 -0
- data/lib/eye/dsl/opts.rb +151 -0
- data/lib/eye/dsl/process_opts.rb +36 -0
- data/lib/eye/dsl/pure_opts.rb +121 -0
- data/lib/eye/dsl/validation.rb +88 -0
- data/lib/eye/group.rb +140 -0
- data/lib/eye/group/chain.rb +81 -0
- data/lib/eye/loader.rb +10 -0
- data/lib/eye/local.rb +100 -0
- data/lib/eye/logger.rb +104 -0
- data/lib/eye/notify.rb +118 -0
- data/lib/eye/notify/jabber.rb +30 -0
- data/lib/eye/notify/mail.rb +48 -0
- data/lib/eye/process.rb +85 -0
- data/lib/eye/process/children.rb +60 -0
- data/lib/eye/process/commands.rb +280 -0
- data/lib/eye/process/config.rb +81 -0
- data/lib/eye/process/controller.rb +73 -0
- data/lib/eye/process/data.rb +78 -0
- data/lib/eye/process/monitor.rb +108 -0
- data/lib/eye/process/notify.rb +32 -0
- data/lib/eye/process/scheduler.rb +82 -0
- data/lib/eye/process/states.rb +86 -0
- data/lib/eye/process/states_history.rb +66 -0
- data/lib/eye/process/system.rb +97 -0
- data/lib/eye/process/trigger.rb +34 -0
- data/lib/eye/process/validate.rb +33 -0
- data/lib/eye/process/watchers.rb +66 -0
- data/lib/eye/reason.rb +20 -0
- data/lib/eye/server.rb +60 -0
- data/lib/eye/sigar.rb +5 -0
- data/lib/eye/system.rb +139 -0
- data/lib/eye/system_resources.rb +99 -0
- data/lib/eye/trigger.rb +136 -0
- data/lib/eye/trigger/check_dependency.rb +30 -0
- data/lib/eye/trigger/flapping.rb +41 -0
- data/lib/eye/trigger/stop_children.rb +17 -0
- data/lib/eye/trigger/transition.rb +15 -0
- data/lib/eye/trigger/wait_dependency.rb +49 -0
- data/lib/eye/utils.rb +45 -0
- data/lib/eye/utils/alive_array.rb +57 -0
- data/lib/eye/utils/celluloid_chain.rb +71 -0
- data/lib/eye/utils/celluloid_klass.rb +5 -0
- data/lib/eye/utils/leak_19.rb +10 -0
- data/lib/eye/utils/mini_active_support.rb +111 -0
- data/lib/eye/utils/pmap.rb +7 -0
- data/lib/eye/utils/tail.rb +20 -0
- metadata +398 -0
@@ -0,0 +1,44 @@
|
|
1
|
+
class Eye::Checker::ChildrenCount < Eye::Checker::Measure
|
2
|
+
|
3
|
+
# check :children_count, :every => 30.seconds, :below => 10, :strategy => :kill_old
|
4
|
+
# monitor_children should be enabled
|
5
|
+
|
6
|
+
param :strategy, Symbol, nil, :restart, [:restart, :kill_old, :kill_new]
|
7
|
+
|
8
|
+
def get_value
|
9
|
+
process.children.size
|
10
|
+
end
|
11
|
+
|
12
|
+
def fire
|
13
|
+
if strategy == :restart
|
14
|
+
super
|
15
|
+
else
|
16
|
+
pids = ordered_by_date_children_pids
|
17
|
+
|
18
|
+
pids = if strategy == :kill_old
|
19
|
+
pids[0...-below]
|
20
|
+
else
|
21
|
+
pids[below..-1]
|
22
|
+
end
|
23
|
+
|
24
|
+
kill_pids(pids)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def kill_pids(pids)
|
31
|
+
info "killing pids: #{pids.inspect} for strategy: #{strategy}"
|
32
|
+
pids.each do |pid|
|
33
|
+
if child = process.children[pid]
|
34
|
+
child.schedule :stop, Eye::Reason.new("bounded #{check_name}")
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def ordered_by_date_children_pids
|
40
|
+
children = process.children.values
|
41
|
+
children.sort_by { |ch| Eye::SystemResources.start_time(ch.pid).to_i }.map &:pid
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
class Eye::Checker::ChildrenMemory < Eye::Checker::Measure
|
2
|
+
|
3
|
+
# check :children_memory, :every => 30.seconds, :below => 400.megabytes
|
4
|
+
# monitor_children should be enabled
|
5
|
+
|
6
|
+
def get_value
|
7
|
+
process.children.values.inject(0) do |sum, ch|
|
8
|
+
sum + Eye::SystemResources.memory(ch.pid).to_i
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
class Eye::Checker::Cpu < Eye::Checker::Measure
|
2
|
+
|
3
|
+
# check :cpu, :every => 3.seconds, :below => 80, :times => [3,5]
|
4
|
+
|
5
|
+
def check_name
|
6
|
+
@check_name ||= "cpu(#{measure_str})"
|
7
|
+
end
|
8
|
+
|
9
|
+
def get_value
|
10
|
+
Eye::SystemResources.cpu(@pid).to_i # nil => 0
|
11
|
+
end
|
12
|
+
|
13
|
+
def human_value(value)
|
14
|
+
"#{value}%"
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
class Eye::Checker::FileCTime < Eye::Checker
|
2
|
+
|
3
|
+
# Check that file changes (log for example)
|
4
|
+
# check :ctime, :every => 5.seconds, :file => "/tmp/1.log", :times => [3,5]
|
5
|
+
|
6
|
+
param :file, [String], true
|
7
|
+
|
8
|
+
def get_value
|
9
|
+
File.ctime(file) rescue nil
|
10
|
+
end
|
11
|
+
|
12
|
+
def human_value(value)
|
13
|
+
if value == nil
|
14
|
+
'Err'
|
15
|
+
else
|
16
|
+
value.strftime('%H:%M')
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def good?(value)
|
21
|
+
value.to_i > previous_value.to_i
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
class Eye::Checker::FileSize < Eye::Checker::Measure
|
2
|
+
|
3
|
+
# Check that file size changed (log for example)
|
4
|
+
# check :fsize, :every => 5.seconds, :file => "/tmp/1.log", :times => [3,5],
|
5
|
+
# :below => 30.kilobytes, :above => 10.kilobytes
|
6
|
+
|
7
|
+
param :file, [String], true
|
8
|
+
|
9
|
+
def check_name
|
10
|
+
@check_name ||= "fsize(#{measure_str})"
|
11
|
+
end
|
12
|
+
|
13
|
+
def get_value
|
14
|
+
File.size(file) rescue nil
|
15
|
+
end
|
16
|
+
|
17
|
+
def human_value(value)
|
18
|
+
"#{value.to_i / 1024}Kb"
|
19
|
+
end
|
20
|
+
|
21
|
+
def good?(value)
|
22
|
+
return true unless previous_value
|
23
|
+
|
24
|
+
diff = value.to_i - previous_value.to_i
|
25
|
+
|
26
|
+
return true if diff < 0 # case when logger nulled
|
27
|
+
|
28
|
+
return false unless super(diff)
|
29
|
+
return false if diff == 0
|
30
|
+
|
31
|
+
true
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
class Eye::Checker::FileTouched < Eye::Checker
|
2
|
+
|
3
|
+
param :file, [String], true
|
4
|
+
param :delete, [TrueClass, FalseClass]
|
5
|
+
|
6
|
+
def get_value
|
7
|
+
File.exists?(file)
|
8
|
+
end
|
9
|
+
|
10
|
+
def good?(value)
|
11
|
+
File.delete(file) if value && delete
|
12
|
+
!value
|
13
|
+
end
|
14
|
+
|
15
|
+
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
|
3
|
+
class Eye::Checker::Http < Eye::Checker::Defer
|
4
|
+
|
5
|
+
# check :http, :every => 5.seconds, :times => 1,
|
6
|
+
# :url => "http://127.0.0.1:3000/", :kind => :success, :pattern => /OK/, :timeout => 3.seconds
|
7
|
+
|
8
|
+
param :url, String, true
|
9
|
+
param :pattern, [String, Regexp]
|
10
|
+
param :kind, [String, Fixnum, Symbol]
|
11
|
+
param :timeout, [Fixnum, Float]
|
12
|
+
param :open_timeout, [Fixnum, Float]
|
13
|
+
param :read_timeout, [Fixnum, Float]
|
14
|
+
|
15
|
+
attr_reader :uri
|
16
|
+
|
17
|
+
def initialize(*args)
|
18
|
+
super
|
19
|
+
|
20
|
+
@uri = URI.parse(url)
|
21
|
+
@kind = case kind
|
22
|
+
when Fixnum then Net::HTTPResponse::CODE_TO_OBJ[kind]
|
23
|
+
when String, Symbol then Net.const_get("HTTP#{kind.to_s.camelize}") rescue Net::HTTPSuccess
|
24
|
+
else
|
25
|
+
Net::HTTPSuccess
|
26
|
+
end
|
27
|
+
@open_timeout = (open_timeout || 3).to_f
|
28
|
+
@read_timeout = (read_timeout || timeout || 15).to_f
|
29
|
+
end
|
30
|
+
|
31
|
+
def get_value
|
32
|
+
res = session.start{ |http| http.get(@uri.request_uri) }
|
33
|
+
{:result => res}
|
34
|
+
|
35
|
+
rescue Timeout::Error => ex
|
36
|
+
debug ex.inspect
|
37
|
+
|
38
|
+
if defined?(Net::OpenTimeout) # for ruby 2.0
|
39
|
+
mes = ex.class.is_a?(Net::OpenTimeout) ? "OpenTimeout<#{@open_timeout}>" : "ReadTimeout<#{@read_timeout}>"
|
40
|
+
{:exception => mes}
|
41
|
+
else
|
42
|
+
{:exception => "Timeout<#{@open_timeout},#{@read_timeout}>"}
|
43
|
+
end
|
44
|
+
|
45
|
+
rescue => ex
|
46
|
+
{:exception => "Error<#{ex.message}>"}
|
47
|
+
end
|
48
|
+
|
49
|
+
def good?(value)
|
50
|
+
return false unless value[:result]
|
51
|
+
|
52
|
+
unless value[:result].kind_of?(@kind)
|
53
|
+
return false
|
54
|
+
end
|
55
|
+
|
56
|
+
if pattern
|
57
|
+
matched = if pattern.is_a?(Regexp)
|
58
|
+
pattern === value[:result].body
|
59
|
+
else
|
60
|
+
value[:result].body.include?(pattern.to_s)
|
61
|
+
end
|
62
|
+
value[:notice] = "missing '#{pattern.to_s}'" unless matched
|
63
|
+
matched
|
64
|
+
else
|
65
|
+
true
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def human_value(value)
|
70
|
+
if !value.is_a?(Hash)
|
71
|
+
'-'
|
72
|
+
elsif value[:exception]
|
73
|
+
value[:exception]
|
74
|
+
else
|
75
|
+
body_size = value[:result].body.size / 1024
|
76
|
+
msg = "#{value[:result].code}=#{body_size}Kb"
|
77
|
+
msg += "<#{value[:notice]}>" if value[:notice]
|
78
|
+
msg
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
private
|
83
|
+
|
84
|
+
def session
|
85
|
+
Net::HTTP.new(@uri.host, @uri.port).tap do |session|
|
86
|
+
if @uri.scheme == 'https'
|
87
|
+
require 'net/https'
|
88
|
+
session.use_ssl = true
|
89
|
+
session.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
90
|
+
end
|
91
|
+
session.open_timeout = @open_timeout
|
92
|
+
session.read_timeout = @read_timeout
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
class Eye::Checker::Memory < Eye::Checker::Measure
|
2
|
+
|
3
|
+
# check :memory, :every => 3.seconds, :below => 80.megabytes, :times => [3,5]
|
4
|
+
|
5
|
+
def check_name
|
6
|
+
@check_name ||= "memory(#{measure_str})"
|
7
|
+
end
|
8
|
+
|
9
|
+
def get_value
|
10
|
+
Eye::SystemResources.memory(@pid).to_i
|
11
|
+
end
|
12
|
+
|
13
|
+
def human_value(value)
|
14
|
+
"#{value.to_i / 1024 / 1024}Mb"
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
class Eye::Checker::Runtime < Eye::Checker::Measure
|
2
|
+
|
3
|
+
# check :runtime, :every => 1.minute, :below => 120.minutes
|
4
|
+
|
5
|
+
def get_value
|
6
|
+
st = Eye::SystemResources.start_time(@pid)
|
7
|
+
if st
|
8
|
+
Time.now.to_i - st.to_i
|
9
|
+
else
|
10
|
+
0
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def human_value(value)
|
15
|
+
"#{value / 60}m"
|
16
|
+
end
|
17
|
+
|
18
|
+
end
|
@@ -0,0 +1,159 @@
|
|
1
|
+
class Eye::Checker::Socket < Eye::Checker::Defer
|
2
|
+
|
3
|
+
# check :socket, :every => 5.seconds, :times => 1,
|
4
|
+
# :addr => "unix:/var/run/daemon.sock", :timeout => 3.seconds,
|
5
|
+
#
|
6
|
+
# Available parameters:
|
7
|
+
# :addr the socket addr to open. The format is tcp://<host>:<port> or unix:<path>
|
8
|
+
# :timeout generic timeout for reading data from socket
|
9
|
+
# :open_timeout override generic timeout for the connection
|
10
|
+
# :read_timeout override generic timeout for data read/write
|
11
|
+
# :send_data after connection send this data
|
12
|
+
# :expect_data after sending :send_data expect this response. Can be a string, Regexp or a Proc
|
13
|
+
# :protocol way of pack,unpack messages (default = socket default), example: :protocol => :em_object
|
14
|
+
|
15
|
+
param :addr, String, true
|
16
|
+
param :timeout, [Fixnum, Float]
|
17
|
+
param :open_timeout, [Fixnum, Float]
|
18
|
+
param :read_timeout, [Fixnum, Float]
|
19
|
+
param :send_data
|
20
|
+
param :expect_data, [String, Regexp, Proc]
|
21
|
+
param :protocol, [Symbol], nil, nil, [:default, :em_object, :raw]
|
22
|
+
|
23
|
+
def initialize(*args)
|
24
|
+
super
|
25
|
+
@open_timeout = (open_timeout || 1).to_f
|
26
|
+
@read_timeout = (read_timeout || timeout || 5).to_f
|
27
|
+
|
28
|
+
if addr =~ %r[\Atcp://(.*?):(.*?)\z]
|
29
|
+
@socket_family = :tcp
|
30
|
+
@socket_addr = $1
|
31
|
+
@socket_port = $2.to_i
|
32
|
+
elsif addr =~ %r[\Aunix:(.*)\z]
|
33
|
+
@socket_family = :unix
|
34
|
+
@socket_path = $1
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def get_value
|
39
|
+
sock = begin
|
40
|
+
Timeout::timeout(@open_timeout){ open_socket }
|
41
|
+
rescue Timeout::Error
|
42
|
+
return { :exception => "OpenTimeout<#{@open_timeout}>" }
|
43
|
+
end
|
44
|
+
|
45
|
+
if send_data
|
46
|
+
begin
|
47
|
+
Timeout::timeout(@read_timeout) do
|
48
|
+
_write_data(sock, send_data)
|
49
|
+
result = _read_data(sock)
|
50
|
+
|
51
|
+
{ :result => result }
|
52
|
+
end
|
53
|
+
rescue Timeout::Error
|
54
|
+
if protocol == :raw
|
55
|
+
return { :result => @buffer }
|
56
|
+
else
|
57
|
+
return { :exception => "ReadTimeout<#{@read_timeout}>" }
|
58
|
+
end
|
59
|
+
end
|
60
|
+
else
|
61
|
+
{ :result => :listen }
|
62
|
+
end
|
63
|
+
|
64
|
+
rescue Exception => e
|
65
|
+
{ :exception => "Error<#{e.message}>" }
|
66
|
+
|
67
|
+
ensure
|
68
|
+
sock.close if sock
|
69
|
+
end
|
70
|
+
|
71
|
+
def good?(value)
|
72
|
+
return false if !value[:result]
|
73
|
+
|
74
|
+
if expect_data
|
75
|
+
if expect_data.is_a?(Proc)
|
76
|
+
match = begin
|
77
|
+
!!expect_data[value[:result]]
|
78
|
+
rescue Timeout::Error, Exception => ex
|
79
|
+
mes = "proc match failed with '#{ex.message}'"
|
80
|
+
error(mes)
|
81
|
+
value[:notice] = mes
|
82
|
+
return false
|
83
|
+
end
|
84
|
+
|
85
|
+
unless match
|
86
|
+
warn "proc #{expect_data} not matched (#{value[:result].truncate(30)}) answer"
|
87
|
+
value[:notice] = 'missing proc validation'
|
88
|
+
end
|
89
|
+
|
90
|
+
return match
|
91
|
+
end
|
92
|
+
|
93
|
+
return true if expect_data.is_a?(Regexp) && expect_data.match(value[:result])
|
94
|
+
return true if value[:result].to_s == expect_data.to_s
|
95
|
+
|
96
|
+
warn "#{expect_data} not matched (#{value[:result].truncate(30)}) answer"
|
97
|
+
value[:notice] = "missing '#{expect_data.to_s}'"
|
98
|
+
return false
|
99
|
+
end
|
100
|
+
|
101
|
+
return true
|
102
|
+
end
|
103
|
+
|
104
|
+
def human_value(value)
|
105
|
+
if !value.is_a?(Hash)
|
106
|
+
'-'
|
107
|
+
elsif value[:exception]
|
108
|
+
value[:exception]
|
109
|
+
else
|
110
|
+
if value[:result] == :listen
|
111
|
+
'listen'
|
112
|
+
else
|
113
|
+
res = "#{value[:result].to_s.size}b"
|
114
|
+
res += "<#{value[:notice]}>" if value[:notice]
|
115
|
+
res
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
private
|
121
|
+
|
122
|
+
def open_socket
|
123
|
+
if @socket_family == :tcp
|
124
|
+
TCPSocket.open(@socket_addr, @socket_port)
|
125
|
+
elsif @socket_family == :unix
|
126
|
+
UNIXSocket.open(@socket_path)
|
127
|
+
else
|
128
|
+
raise "Unknown socket addr #{addr}"
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
def _write_data(socket, data)
|
133
|
+
case protocol
|
134
|
+
when :em_object
|
135
|
+
data = Marshal.dump(data)
|
136
|
+
socket.write([data.bytesize, data].pack('Na*'))
|
137
|
+
else
|
138
|
+
socket.write(data.to_s)
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
def _read_data(socket)
|
143
|
+
case protocol
|
144
|
+
when :em_object
|
145
|
+
content = ''
|
146
|
+
msg_size = socket.recv(4).unpack('N')[0] rescue 0
|
147
|
+
content << socket.recv(msg_size - content.length) while content.length < msg_size
|
148
|
+
if content.present?
|
149
|
+
Marshal.load(content) rescue 'corrupted_marshal'
|
150
|
+
end
|
151
|
+
when :raw
|
152
|
+
@buffer = ''
|
153
|
+
loop { @buffer << socket.recv(1) }
|
154
|
+
else
|
155
|
+
socket.readline.chop
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
end
|