ace-eye 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/.gitignore +38 -0
- data/.rspec +2 -0
- data/.travis.yml +6 -0
- data/CHANGES.md +77 -0
- data/Gemfile +6 -0
- data/LICENSE +22 -0
- data/README.md +212 -0
- data/Rakefile +35 -0
- data/bin/eye +5 -0
- data/bin/loader_eye +72 -0
- data/bin/runner +16 -0
- data/examples/dependency.eye +17 -0
- data/examples/notify.eye +19 -0
- data/examples/plugin/README.md +15 -0
- data/examples/plugin/main.eye +15 -0
- data/examples/plugin/plugin.rb +63 -0
- data/examples/process_thin.rb +29 -0
- data/examples/processes/em.rb +57 -0
- data/examples/processes/forking.rb +20 -0
- data/examples/processes/sample.rb +144 -0
- data/examples/processes/thin.ru +12 -0
- data/examples/puma.eye +29 -0
- data/examples/rbenv.eye +11 -0
- data/examples/sidekiq.eye +23 -0
- data/examples/test.eye +87 -0
- data/examples/thin-farm.eye +30 -0
- data/examples/unicorn.eye +39 -0
- data/eye.gemspec +40 -0
- data/lib/eye.rb +28 -0
- data/lib/eye/application.rb +73 -0
- data/lib/eye/checker.rb +258 -0
- data/lib/eye/checker/children_count.rb +44 -0
- data/lib/eye/checker/children_memory.rb +12 -0
- data/lib/eye/checker/cpu.rb +17 -0
- data/lib/eye/checker/cputime.rb +13 -0
- data/lib/eye/checker/file_ctime.rb +24 -0
- data/lib/eye/checker/file_size.rb +34 -0
- data/lib/eye/checker/file_touched.rb +15 -0
- data/lib/eye/checker/http.rb +96 -0
- data/lib/eye/checker/memory.rb +17 -0
- data/lib/eye/checker/nop.rb +6 -0
- data/lib/eye/checker/runtime.rb +18 -0
- data/lib/eye/checker/socket.rb +159 -0
- data/lib/eye/child_process.rb +101 -0
- data/lib/eye/cli.rb +185 -0
- data/lib/eye/cli/commands.rb +78 -0
- data/lib/eye/cli/render.rb +130 -0
- data/lib/eye/cli/server.rb +93 -0
- data/lib/eye/client.rb +32 -0
- data/lib/eye/config.rb +91 -0
- data/lib/eye/control.rb +2 -0
- data/lib/eye/controller.rb +54 -0
- data/lib/eye/controller/commands.rb +88 -0
- data/lib/eye/controller/helpers.rb +101 -0
- data/lib/eye/controller/load.rb +224 -0
- data/lib/eye/controller/options.rb +18 -0
- data/lib/eye/controller/send_command.rb +177 -0
- data/lib/eye/controller/status.rb +72 -0
- data/lib/eye/dsl.rb +53 -0
- data/lib/eye/dsl/application_opts.rb +39 -0
- data/lib/eye/dsl/chain.rb +12 -0
- data/lib/eye/dsl/child_process_opts.rb +13 -0
- data/lib/eye/dsl/config_opts.rb +55 -0
- data/lib/eye/dsl/group_opts.rb +32 -0
- data/lib/eye/dsl/helpers.rb +20 -0
- data/lib/eye/dsl/main.rb +51 -0
- data/lib/eye/dsl/opts.rb +151 -0
- data/lib/eye/dsl/process_opts.rb +36 -0
- data/lib/eye/dsl/pure_opts.rb +121 -0
- data/lib/eye/dsl/validation.rb +88 -0
- data/lib/eye/group.rb +140 -0
- data/lib/eye/group/chain.rb +81 -0
- data/lib/eye/loader.rb +10 -0
- data/lib/eye/local.rb +100 -0
- data/lib/eye/logger.rb +104 -0
- data/lib/eye/notify.rb +118 -0
- data/lib/eye/notify/jabber.rb +30 -0
- data/lib/eye/notify/mail.rb +48 -0
- data/lib/eye/process.rb +85 -0
- data/lib/eye/process/children.rb +60 -0
- data/lib/eye/process/commands.rb +280 -0
- data/lib/eye/process/config.rb +81 -0
- data/lib/eye/process/controller.rb +73 -0
- data/lib/eye/process/data.rb +78 -0
- data/lib/eye/process/monitor.rb +108 -0
- data/lib/eye/process/notify.rb +32 -0
- data/lib/eye/process/scheduler.rb +82 -0
- data/lib/eye/process/states.rb +86 -0
- data/lib/eye/process/states_history.rb +66 -0
- data/lib/eye/process/system.rb +97 -0
- data/lib/eye/process/trigger.rb +34 -0
- data/lib/eye/process/validate.rb +33 -0
- data/lib/eye/process/watchers.rb +66 -0
- data/lib/eye/reason.rb +20 -0
- data/lib/eye/server.rb +60 -0
- data/lib/eye/sigar.rb +5 -0
- data/lib/eye/system.rb +139 -0
- data/lib/eye/system_resources.rb +99 -0
- data/lib/eye/trigger.rb +136 -0
- data/lib/eye/trigger/check_dependency.rb +30 -0
- data/lib/eye/trigger/flapping.rb +41 -0
- data/lib/eye/trigger/stop_children.rb +17 -0
- data/lib/eye/trigger/transition.rb +15 -0
- data/lib/eye/trigger/wait_dependency.rb +49 -0
- data/lib/eye/utils.rb +45 -0
- data/lib/eye/utils/alive_array.rb +57 -0
- data/lib/eye/utils/celluloid_chain.rb +71 -0
- data/lib/eye/utils/celluloid_klass.rb +5 -0
- data/lib/eye/utils/leak_19.rb +10 -0
- data/lib/eye/utils/mini_active_support.rb +111 -0
- data/lib/eye/utils/pmap.rb +7 -0
- data/lib/eye/utils/tail.rb +20 -0
- metadata +398 -0
@@ -0,0 +1,44 @@
|
|
1
|
+
class Eye::Checker::ChildrenCount < Eye::Checker::Measure
|
2
|
+
|
3
|
+
# check :children_count, :every => 30.seconds, :below => 10, :strategy => :kill_old
|
4
|
+
# monitor_children should be enabled
|
5
|
+
|
6
|
+
param :strategy, Symbol, nil, :restart, [:restart, :kill_old, :kill_new]
|
7
|
+
|
8
|
+
def get_value
|
9
|
+
process.children.size
|
10
|
+
end
|
11
|
+
|
12
|
+
def fire
|
13
|
+
if strategy == :restart
|
14
|
+
super
|
15
|
+
else
|
16
|
+
pids = ordered_by_date_children_pids
|
17
|
+
|
18
|
+
pids = if strategy == :kill_old
|
19
|
+
pids[0...-below]
|
20
|
+
else
|
21
|
+
pids[below..-1]
|
22
|
+
end
|
23
|
+
|
24
|
+
kill_pids(pids)
|
25
|
+
end
|
26
|
+
end
|
27
|
+
|
28
|
+
private
|
29
|
+
|
30
|
+
def kill_pids(pids)
|
31
|
+
info "killing pids: #{pids.inspect} for strategy: #{strategy}"
|
32
|
+
pids.each do |pid|
|
33
|
+
if child = process.children[pid]
|
34
|
+
child.schedule :stop, Eye::Reason.new("bounded #{check_name}")
|
35
|
+
end
|
36
|
+
end
|
37
|
+
end
|
38
|
+
|
39
|
+
def ordered_by_date_children_pids
|
40
|
+
children = process.children.values
|
41
|
+
children.sort_by { |ch| Eye::SystemResources.start_time(ch.pid).to_i }.map &:pid
|
42
|
+
end
|
43
|
+
|
44
|
+
end
|
@@ -0,0 +1,12 @@
|
|
1
|
+
class Eye::Checker::ChildrenMemory < Eye::Checker::Measure
|
2
|
+
|
3
|
+
# check :children_memory, :every => 30.seconds, :below => 400.megabytes
|
4
|
+
# monitor_children should be enabled
|
5
|
+
|
6
|
+
def get_value
|
7
|
+
process.children.values.inject(0) do |sum, ch|
|
8
|
+
sum + Eye::SystemResources.memory(ch.pid).to_i
|
9
|
+
end
|
10
|
+
end
|
11
|
+
|
12
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
class Eye::Checker::Cpu < Eye::Checker::Measure
|
2
|
+
|
3
|
+
# check :cpu, :every => 3.seconds, :below => 80, :times => [3,5]
|
4
|
+
|
5
|
+
def check_name
|
6
|
+
@check_name ||= "cpu(#{measure_str})"
|
7
|
+
end
|
8
|
+
|
9
|
+
def get_value
|
10
|
+
Eye::SystemResources.cpu(@pid).to_i # nil => 0
|
11
|
+
end
|
12
|
+
|
13
|
+
def human_value(value)
|
14
|
+
"#{value}%"
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
class Eye::Checker::FileCTime < Eye::Checker
|
2
|
+
|
3
|
+
# Check that file changes (log for example)
|
4
|
+
# check :ctime, :every => 5.seconds, :file => "/tmp/1.log", :times => [3,5]
|
5
|
+
|
6
|
+
param :file, [String], true
|
7
|
+
|
8
|
+
def get_value
|
9
|
+
File.ctime(file) rescue nil
|
10
|
+
end
|
11
|
+
|
12
|
+
def human_value(value)
|
13
|
+
if value == nil
|
14
|
+
'Err'
|
15
|
+
else
|
16
|
+
value.strftime('%H:%M')
|
17
|
+
end
|
18
|
+
end
|
19
|
+
|
20
|
+
def good?(value)
|
21
|
+
value.to_i > previous_value.to_i
|
22
|
+
end
|
23
|
+
|
24
|
+
end
|
@@ -0,0 +1,34 @@
|
|
1
|
+
class Eye::Checker::FileSize < Eye::Checker::Measure
|
2
|
+
|
3
|
+
# Check that file size changed (log for example)
|
4
|
+
# check :fsize, :every => 5.seconds, :file => "/tmp/1.log", :times => [3,5],
|
5
|
+
# :below => 30.kilobytes, :above => 10.kilobytes
|
6
|
+
|
7
|
+
param :file, [String], true
|
8
|
+
|
9
|
+
def check_name
|
10
|
+
@check_name ||= "fsize(#{measure_str})"
|
11
|
+
end
|
12
|
+
|
13
|
+
def get_value
|
14
|
+
File.size(file) rescue nil
|
15
|
+
end
|
16
|
+
|
17
|
+
def human_value(value)
|
18
|
+
"#{value.to_i / 1024}Kb"
|
19
|
+
end
|
20
|
+
|
21
|
+
def good?(value)
|
22
|
+
return true unless previous_value
|
23
|
+
|
24
|
+
diff = value.to_i - previous_value.to_i
|
25
|
+
|
26
|
+
return true if diff < 0 # case when logger nulled
|
27
|
+
|
28
|
+
return false unless super(diff)
|
29
|
+
return false if diff == 0
|
30
|
+
|
31
|
+
true
|
32
|
+
end
|
33
|
+
|
34
|
+
end
|
@@ -0,0 +1,15 @@
|
|
1
|
+
class Eye::Checker::FileTouched < Eye::Checker
|
2
|
+
|
3
|
+
param :file, [String], true
|
4
|
+
param :delete, [TrueClass, FalseClass]
|
5
|
+
|
6
|
+
def get_value
|
7
|
+
File.exists?(file)
|
8
|
+
end
|
9
|
+
|
10
|
+
def good?(value)
|
11
|
+
File.delete(file) if value && delete
|
12
|
+
!value
|
13
|
+
end
|
14
|
+
|
15
|
+
end
|
@@ -0,0 +1,96 @@
|
|
1
|
+
require 'net/http'
|
2
|
+
|
3
|
+
class Eye::Checker::Http < Eye::Checker::Defer
|
4
|
+
|
5
|
+
# check :http, :every => 5.seconds, :times => 1,
|
6
|
+
# :url => "http://127.0.0.1:3000/", :kind => :success, :pattern => /OK/, :timeout => 3.seconds
|
7
|
+
|
8
|
+
param :url, String, true
|
9
|
+
param :pattern, [String, Regexp]
|
10
|
+
param :kind, [String, Fixnum, Symbol]
|
11
|
+
param :timeout, [Fixnum, Float]
|
12
|
+
param :open_timeout, [Fixnum, Float]
|
13
|
+
param :read_timeout, [Fixnum, Float]
|
14
|
+
|
15
|
+
attr_reader :uri
|
16
|
+
|
17
|
+
def initialize(*args)
|
18
|
+
super
|
19
|
+
|
20
|
+
@uri = URI.parse(url)
|
21
|
+
@kind = case kind
|
22
|
+
when Fixnum then Net::HTTPResponse::CODE_TO_OBJ[kind]
|
23
|
+
when String, Symbol then Net.const_get("HTTP#{kind.to_s.camelize}") rescue Net::HTTPSuccess
|
24
|
+
else
|
25
|
+
Net::HTTPSuccess
|
26
|
+
end
|
27
|
+
@open_timeout = (open_timeout || 3).to_f
|
28
|
+
@read_timeout = (read_timeout || timeout || 15).to_f
|
29
|
+
end
|
30
|
+
|
31
|
+
def get_value
|
32
|
+
res = session.start{ |http| http.get(@uri.request_uri) }
|
33
|
+
{:result => res}
|
34
|
+
|
35
|
+
rescue Timeout::Error => ex
|
36
|
+
debug ex.inspect
|
37
|
+
|
38
|
+
if defined?(Net::OpenTimeout) # for ruby 2.0
|
39
|
+
mes = ex.class.is_a?(Net::OpenTimeout) ? "OpenTimeout<#{@open_timeout}>" : "ReadTimeout<#{@read_timeout}>"
|
40
|
+
{:exception => mes}
|
41
|
+
else
|
42
|
+
{:exception => "Timeout<#{@open_timeout},#{@read_timeout}>"}
|
43
|
+
end
|
44
|
+
|
45
|
+
rescue => ex
|
46
|
+
{:exception => "Error<#{ex.message}>"}
|
47
|
+
end
|
48
|
+
|
49
|
+
def good?(value)
|
50
|
+
return false unless value[:result]
|
51
|
+
|
52
|
+
unless value[:result].kind_of?(@kind)
|
53
|
+
return false
|
54
|
+
end
|
55
|
+
|
56
|
+
if pattern
|
57
|
+
matched = if pattern.is_a?(Regexp)
|
58
|
+
pattern === value[:result].body
|
59
|
+
else
|
60
|
+
value[:result].body.include?(pattern.to_s)
|
61
|
+
end
|
62
|
+
value[:notice] = "missing '#{pattern.to_s}'" unless matched
|
63
|
+
matched
|
64
|
+
else
|
65
|
+
true
|
66
|
+
end
|
67
|
+
end
|
68
|
+
|
69
|
+
def human_value(value)
|
70
|
+
if !value.is_a?(Hash)
|
71
|
+
'-'
|
72
|
+
elsif value[:exception]
|
73
|
+
value[:exception]
|
74
|
+
else
|
75
|
+
body_size = value[:result].body.size / 1024
|
76
|
+
msg = "#{value[:result].code}=#{body_size}Kb"
|
77
|
+
msg += "<#{value[:notice]}>" if value[:notice]
|
78
|
+
msg
|
79
|
+
end
|
80
|
+
end
|
81
|
+
|
82
|
+
private
|
83
|
+
|
84
|
+
def session
|
85
|
+
Net::HTTP.new(@uri.host, @uri.port).tap do |session|
|
86
|
+
if @uri.scheme == 'https'
|
87
|
+
require 'net/https'
|
88
|
+
session.use_ssl = true
|
89
|
+
session.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
90
|
+
end
|
91
|
+
session.open_timeout = @open_timeout
|
92
|
+
session.read_timeout = @read_timeout
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
end
|
@@ -0,0 +1,17 @@
|
|
1
|
+
class Eye::Checker::Memory < Eye::Checker::Measure
|
2
|
+
|
3
|
+
# check :memory, :every => 3.seconds, :below => 80.megabytes, :times => [3,5]
|
4
|
+
|
5
|
+
def check_name
|
6
|
+
@check_name ||= "memory(#{measure_str})"
|
7
|
+
end
|
8
|
+
|
9
|
+
def get_value
|
10
|
+
Eye::SystemResources.memory(@pid).to_i
|
11
|
+
end
|
12
|
+
|
13
|
+
def human_value(value)
|
14
|
+
"#{value.to_i / 1024 / 1024}Mb"
|
15
|
+
end
|
16
|
+
|
17
|
+
end
|
@@ -0,0 +1,18 @@
|
|
1
|
+
class Eye::Checker::Runtime < Eye::Checker::Measure
|
2
|
+
|
3
|
+
# check :runtime, :every => 1.minute, :below => 120.minutes
|
4
|
+
|
5
|
+
def get_value
|
6
|
+
st = Eye::SystemResources.start_time(@pid)
|
7
|
+
if st
|
8
|
+
Time.now.to_i - st.to_i
|
9
|
+
else
|
10
|
+
0
|
11
|
+
end
|
12
|
+
end
|
13
|
+
|
14
|
+
def human_value(value)
|
15
|
+
"#{value / 60}m"
|
16
|
+
end
|
17
|
+
|
18
|
+
end
|
@@ -0,0 +1,159 @@
|
|
1
|
+
class Eye::Checker::Socket < Eye::Checker::Defer
|
2
|
+
|
3
|
+
# check :socket, :every => 5.seconds, :times => 1,
|
4
|
+
# :addr => "unix:/var/run/daemon.sock", :timeout => 3.seconds,
|
5
|
+
#
|
6
|
+
# Available parameters:
|
7
|
+
# :addr the socket addr to open. The format is tcp://<host>:<port> or unix:<path>
|
8
|
+
# :timeout generic timeout for reading data from socket
|
9
|
+
# :open_timeout override generic timeout for the connection
|
10
|
+
# :read_timeout override generic timeout for data read/write
|
11
|
+
# :send_data after connection send this data
|
12
|
+
# :expect_data after sending :send_data expect this response. Can be a string, Regexp or a Proc
|
13
|
+
# :protocol way of pack,unpack messages (default = socket default), example: :protocol => :em_object
|
14
|
+
|
15
|
+
param :addr, String, true
|
16
|
+
param :timeout, [Fixnum, Float]
|
17
|
+
param :open_timeout, [Fixnum, Float]
|
18
|
+
param :read_timeout, [Fixnum, Float]
|
19
|
+
param :send_data
|
20
|
+
param :expect_data, [String, Regexp, Proc]
|
21
|
+
param :protocol, [Symbol], nil, nil, [:default, :em_object, :raw]
|
22
|
+
|
23
|
+
def initialize(*args)
|
24
|
+
super
|
25
|
+
@open_timeout = (open_timeout || 1).to_f
|
26
|
+
@read_timeout = (read_timeout || timeout || 5).to_f
|
27
|
+
|
28
|
+
if addr =~ %r[\Atcp://(.*?):(.*?)\z]
|
29
|
+
@socket_family = :tcp
|
30
|
+
@socket_addr = $1
|
31
|
+
@socket_port = $2.to_i
|
32
|
+
elsif addr =~ %r[\Aunix:(.*)\z]
|
33
|
+
@socket_family = :unix
|
34
|
+
@socket_path = $1
|
35
|
+
end
|
36
|
+
end
|
37
|
+
|
38
|
+
def get_value
|
39
|
+
sock = begin
|
40
|
+
Timeout::timeout(@open_timeout){ open_socket }
|
41
|
+
rescue Timeout::Error
|
42
|
+
return { :exception => "OpenTimeout<#{@open_timeout}>" }
|
43
|
+
end
|
44
|
+
|
45
|
+
if send_data
|
46
|
+
begin
|
47
|
+
Timeout::timeout(@read_timeout) do
|
48
|
+
_write_data(sock, send_data)
|
49
|
+
result = _read_data(sock)
|
50
|
+
|
51
|
+
{ :result => result }
|
52
|
+
end
|
53
|
+
rescue Timeout::Error
|
54
|
+
if protocol == :raw
|
55
|
+
return { :result => @buffer }
|
56
|
+
else
|
57
|
+
return { :exception => "ReadTimeout<#{@read_timeout}>" }
|
58
|
+
end
|
59
|
+
end
|
60
|
+
else
|
61
|
+
{ :result => :listen }
|
62
|
+
end
|
63
|
+
|
64
|
+
rescue Exception => e
|
65
|
+
{ :exception => "Error<#{e.message}>" }
|
66
|
+
|
67
|
+
ensure
|
68
|
+
sock.close if sock
|
69
|
+
end
|
70
|
+
|
71
|
+
def good?(value)
|
72
|
+
return false if !value[:result]
|
73
|
+
|
74
|
+
if expect_data
|
75
|
+
if expect_data.is_a?(Proc)
|
76
|
+
match = begin
|
77
|
+
!!expect_data[value[:result]]
|
78
|
+
rescue Timeout::Error, Exception => ex
|
79
|
+
mes = "proc match failed with '#{ex.message}'"
|
80
|
+
error(mes)
|
81
|
+
value[:notice] = mes
|
82
|
+
return false
|
83
|
+
end
|
84
|
+
|
85
|
+
unless match
|
86
|
+
warn "proc #{expect_data} not matched (#{value[:result].truncate(30)}) answer"
|
87
|
+
value[:notice] = 'missing proc validation'
|
88
|
+
end
|
89
|
+
|
90
|
+
return match
|
91
|
+
end
|
92
|
+
|
93
|
+
return true if expect_data.is_a?(Regexp) && expect_data.match(value[:result])
|
94
|
+
return true if value[:result].to_s == expect_data.to_s
|
95
|
+
|
96
|
+
warn "#{expect_data} not matched (#{value[:result].truncate(30)}) answer"
|
97
|
+
value[:notice] = "missing '#{expect_data.to_s}'"
|
98
|
+
return false
|
99
|
+
end
|
100
|
+
|
101
|
+
return true
|
102
|
+
end
|
103
|
+
|
104
|
+
def human_value(value)
|
105
|
+
if !value.is_a?(Hash)
|
106
|
+
'-'
|
107
|
+
elsif value[:exception]
|
108
|
+
value[:exception]
|
109
|
+
else
|
110
|
+
if value[:result] == :listen
|
111
|
+
'listen'
|
112
|
+
else
|
113
|
+
res = "#{value[:result].to_s.size}b"
|
114
|
+
res += "<#{value[:notice]}>" if value[:notice]
|
115
|
+
res
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
119
|
+
|
120
|
+
private
|
121
|
+
|
122
|
+
def open_socket
|
123
|
+
if @socket_family == :tcp
|
124
|
+
TCPSocket.open(@socket_addr, @socket_port)
|
125
|
+
elsif @socket_family == :unix
|
126
|
+
UNIXSocket.open(@socket_path)
|
127
|
+
else
|
128
|
+
raise "Unknown socket addr #{addr}"
|
129
|
+
end
|
130
|
+
end
|
131
|
+
|
132
|
+
def _write_data(socket, data)
|
133
|
+
case protocol
|
134
|
+
when :em_object
|
135
|
+
data = Marshal.dump(data)
|
136
|
+
socket.write([data.bytesize, data].pack('Na*'))
|
137
|
+
else
|
138
|
+
socket.write(data.to_s)
|
139
|
+
end
|
140
|
+
end
|
141
|
+
|
142
|
+
def _read_data(socket)
|
143
|
+
case protocol
|
144
|
+
when :em_object
|
145
|
+
content = ''
|
146
|
+
msg_size = socket.recv(4).unpack('N')[0] rescue 0
|
147
|
+
content << socket.recv(msg_size - content.length) while content.length < msg_size
|
148
|
+
if content.present?
|
149
|
+
Marshal.load(content) rescue 'corrupted_marshal'
|
150
|
+
end
|
151
|
+
when :raw
|
152
|
+
@buffer = ''
|
153
|
+
loop { @buffer << socket.recv(1) }
|
154
|
+
else
|
155
|
+
socket.readline.chop
|
156
|
+
end
|
157
|
+
end
|
158
|
+
|
159
|
+
end
|