ace-eye 0.6.1

Sign up to get free protection for your applications and to get access to all the features.
Files changed (114) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +38 -0
  3. data/.rspec +2 -0
  4. data/.travis.yml +6 -0
  5. data/CHANGES.md +77 -0
  6. data/Gemfile +6 -0
  7. data/LICENSE +22 -0
  8. data/README.md +212 -0
  9. data/Rakefile +35 -0
  10. data/bin/eye +5 -0
  11. data/bin/loader_eye +72 -0
  12. data/bin/runner +16 -0
  13. data/examples/dependency.eye +17 -0
  14. data/examples/notify.eye +19 -0
  15. data/examples/plugin/README.md +15 -0
  16. data/examples/plugin/main.eye +15 -0
  17. data/examples/plugin/plugin.rb +63 -0
  18. data/examples/process_thin.rb +29 -0
  19. data/examples/processes/em.rb +57 -0
  20. data/examples/processes/forking.rb +20 -0
  21. data/examples/processes/sample.rb +144 -0
  22. data/examples/processes/thin.ru +12 -0
  23. data/examples/puma.eye +29 -0
  24. data/examples/rbenv.eye +11 -0
  25. data/examples/sidekiq.eye +23 -0
  26. data/examples/test.eye +87 -0
  27. data/examples/thin-farm.eye +30 -0
  28. data/examples/unicorn.eye +39 -0
  29. data/eye.gemspec +40 -0
  30. data/lib/eye.rb +28 -0
  31. data/lib/eye/application.rb +73 -0
  32. data/lib/eye/checker.rb +258 -0
  33. data/lib/eye/checker/children_count.rb +44 -0
  34. data/lib/eye/checker/children_memory.rb +12 -0
  35. data/lib/eye/checker/cpu.rb +17 -0
  36. data/lib/eye/checker/cputime.rb +13 -0
  37. data/lib/eye/checker/file_ctime.rb +24 -0
  38. data/lib/eye/checker/file_size.rb +34 -0
  39. data/lib/eye/checker/file_touched.rb +15 -0
  40. data/lib/eye/checker/http.rb +96 -0
  41. data/lib/eye/checker/memory.rb +17 -0
  42. data/lib/eye/checker/nop.rb +6 -0
  43. data/lib/eye/checker/runtime.rb +18 -0
  44. data/lib/eye/checker/socket.rb +159 -0
  45. data/lib/eye/child_process.rb +101 -0
  46. data/lib/eye/cli.rb +185 -0
  47. data/lib/eye/cli/commands.rb +78 -0
  48. data/lib/eye/cli/render.rb +130 -0
  49. data/lib/eye/cli/server.rb +93 -0
  50. data/lib/eye/client.rb +32 -0
  51. data/lib/eye/config.rb +91 -0
  52. data/lib/eye/control.rb +2 -0
  53. data/lib/eye/controller.rb +54 -0
  54. data/lib/eye/controller/commands.rb +88 -0
  55. data/lib/eye/controller/helpers.rb +101 -0
  56. data/lib/eye/controller/load.rb +224 -0
  57. data/lib/eye/controller/options.rb +18 -0
  58. data/lib/eye/controller/send_command.rb +177 -0
  59. data/lib/eye/controller/status.rb +72 -0
  60. data/lib/eye/dsl.rb +53 -0
  61. data/lib/eye/dsl/application_opts.rb +39 -0
  62. data/lib/eye/dsl/chain.rb +12 -0
  63. data/lib/eye/dsl/child_process_opts.rb +13 -0
  64. data/lib/eye/dsl/config_opts.rb +55 -0
  65. data/lib/eye/dsl/group_opts.rb +32 -0
  66. data/lib/eye/dsl/helpers.rb +20 -0
  67. data/lib/eye/dsl/main.rb +51 -0
  68. data/lib/eye/dsl/opts.rb +151 -0
  69. data/lib/eye/dsl/process_opts.rb +36 -0
  70. data/lib/eye/dsl/pure_opts.rb +121 -0
  71. data/lib/eye/dsl/validation.rb +88 -0
  72. data/lib/eye/group.rb +140 -0
  73. data/lib/eye/group/chain.rb +81 -0
  74. data/lib/eye/loader.rb +10 -0
  75. data/lib/eye/local.rb +100 -0
  76. data/lib/eye/logger.rb +104 -0
  77. data/lib/eye/notify.rb +118 -0
  78. data/lib/eye/notify/jabber.rb +30 -0
  79. data/lib/eye/notify/mail.rb +48 -0
  80. data/lib/eye/process.rb +85 -0
  81. data/lib/eye/process/children.rb +60 -0
  82. data/lib/eye/process/commands.rb +280 -0
  83. data/lib/eye/process/config.rb +81 -0
  84. data/lib/eye/process/controller.rb +73 -0
  85. data/lib/eye/process/data.rb +78 -0
  86. data/lib/eye/process/monitor.rb +108 -0
  87. data/lib/eye/process/notify.rb +32 -0
  88. data/lib/eye/process/scheduler.rb +82 -0
  89. data/lib/eye/process/states.rb +86 -0
  90. data/lib/eye/process/states_history.rb +66 -0
  91. data/lib/eye/process/system.rb +97 -0
  92. data/lib/eye/process/trigger.rb +34 -0
  93. data/lib/eye/process/validate.rb +33 -0
  94. data/lib/eye/process/watchers.rb +66 -0
  95. data/lib/eye/reason.rb +20 -0
  96. data/lib/eye/server.rb +60 -0
  97. data/lib/eye/sigar.rb +5 -0
  98. data/lib/eye/system.rb +139 -0
  99. data/lib/eye/system_resources.rb +99 -0
  100. data/lib/eye/trigger.rb +136 -0
  101. data/lib/eye/trigger/check_dependency.rb +30 -0
  102. data/lib/eye/trigger/flapping.rb +41 -0
  103. data/lib/eye/trigger/stop_children.rb +17 -0
  104. data/lib/eye/trigger/transition.rb +15 -0
  105. data/lib/eye/trigger/wait_dependency.rb +49 -0
  106. data/lib/eye/utils.rb +45 -0
  107. data/lib/eye/utils/alive_array.rb +57 -0
  108. data/lib/eye/utils/celluloid_chain.rb +71 -0
  109. data/lib/eye/utils/celluloid_klass.rb +5 -0
  110. data/lib/eye/utils/leak_19.rb +10 -0
  111. data/lib/eye/utils/mini_active_support.rb +111 -0
  112. data/lib/eye/utils/pmap.rb +7 -0
  113. data/lib/eye/utils/tail.rb +20 -0
  114. metadata +398 -0
@@ -0,0 +1,44 @@
1
+ class Eye::Checker::ChildrenCount < Eye::Checker::Measure
2
+
3
+ # check :children_count, :every => 30.seconds, :below => 10, :strategy => :kill_old
4
+ # monitor_children should be enabled
5
+
6
+ param :strategy, Symbol, nil, :restart, [:restart, :kill_old, :kill_new]
7
+
8
+ def get_value
9
+ process.children.size
10
+ end
11
+
12
+ def fire
13
+ if strategy == :restart
14
+ super
15
+ else
16
+ pids = ordered_by_date_children_pids
17
+
18
+ pids = if strategy == :kill_old
19
+ pids[0...-below]
20
+ else
21
+ pids[below..-1]
22
+ end
23
+
24
+ kill_pids(pids)
25
+ end
26
+ end
27
+
28
+ private
29
+
30
+ def kill_pids(pids)
31
+ info "killing pids: #{pids.inspect} for strategy: #{strategy}"
32
+ pids.each do |pid|
33
+ if child = process.children[pid]
34
+ child.schedule :stop, Eye::Reason.new("bounded #{check_name}")
35
+ end
36
+ end
37
+ end
38
+
39
+ def ordered_by_date_children_pids
40
+ children = process.children.values
41
+ children.sort_by { |ch| Eye::SystemResources.start_time(ch.pid).to_i }.map &:pid
42
+ end
43
+
44
+ end
@@ -0,0 +1,12 @@
1
+ class Eye::Checker::ChildrenMemory < Eye::Checker::Measure
2
+
3
+ # check :children_memory, :every => 30.seconds, :below => 400.megabytes
4
+ # monitor_children should be enabled
5
+
6
+ def get_value
7
+ process.children.values.inject(0) do |sum, ch|
8
+ sum + Eye::SystemResources.memory(ch.pid).to_i
9
+ end
10
+ end
11
+
12
+ end
@@ -0,0 +1,17 @@
1
+ class Eye::Checker::Cpu < Eye::Checker::Measure
2
+
3
+ # check :cpu, :every => 3.seconds, :below => 80, :times => [3,5]
4
+
5
+ def check_name
6
+ @check_name ||= "cpu(#{measure_str})"
7
+ end
8
+
9
+ def get_value
10
+ Eye::SystemResources.cpu(@pid).to_i # nil => 0
11
+ end
12
+
13
+ def human_value(value)
14
+ "#{value}%"
15
+ end
16
+
17
+ end
@@ -0,0 +1,13 @@
1
+ class Eye::Checker::Cputime < Eye::Checker::Measure
2
+
3
+ # check :cputime, :every => 1.minute, :below => 120.minutes
4
+
5
+ def get_value
6
+ Eye::SystemResources.cputime(@pid).to_f
7
+ end
8
+
9
+ def human_value(value)
10
+ "#{value / 60}m"
11
+ end
12
+
13
+ end
@@ -0,0 +1,24 @@
1
+ class Eye::Checker::FileCTime < Eye::Checker
2
+
3
+ # Check that file changes (log for example)
4
+ # check :ctime, :every => 5.seconds, :file => "/tmp/1.log", :times => [3,5]
5
+
6
+ param :file, [String], true
7
+
8
+ def get_value
9
+ File.ctime(file) rescue nil
10
+ end
11
+
12
+ def human_value(value)
13
+ if value == nil
14
+ 'Err'
15
+ else
16
+ value.strftime('%H:%M')
17
+ end
18
+ end
19
+
20
+ def good?(value)
21
+ value.to_i > previous_value.to_i
22
+ end
23
+
24
+ end
@@ -0,0 +1,34 @@
1
+ class Eye::Checker::FileSize < Eye::Checker::Measure
2
+
3
+ # Check that file size changed (log for example)
4
+ # check :fsize, :every => 5.seconds, :file => "/tmp/1.log", :times => [3,5],
5
+ # :below => 30.kilobytes, :above => 10.kilobytes
6
+
7
+ param :file, [String], true
8
+
9
+ def check_name
10
+ @check_name ||= "fsize(#{measure_str})"
11
+ end
12
+
13
+ def get_value
14
+ File.size(file) rescue nil
15
+ end
16
+
17
+ def human_value(value)
18
+ "#{value.to_i / 1024}Kb"
19
+ end
20
+
21
+ def good?(value)
22
+ return true unless previous_value
23
+
24
+ diff = value.to_i - previous_value.to_i
25
+
26
+ return true if diff < 0 # case when logger nulled
27
+
28
+ return false unless super(diff)
29
+ return false if diff == 0
30
+
31
+ true
32
+ end
33
+
34
+ end
@@ -0,0 +1,15 @@
1
+ class Eye::Checker::FileTouched < Eye::Checker
2
+
3
+ param :file, [String], true
4
+ param :delete, [TrueClass, FalseClass]
5
+
6
+ def get_value
7
+ File.exists?(file)
8
+ end
9
+
10
+ def good?(value)
11
+ File.delete(file) if value && delete
12
+ !value
13
+ end
14
+
15
+ end
@@ -0,0 +1,96 @@
1
+ require 'net/http'
2
+
3
+ class Eye::Checker::Http < Eye::Checker::Defer
4
+
5
+ # check :http, :every => 5.seconds, :times => 1,
6
+ # :url => "http://127.0.0.1:3000/", :kind => :success, :pattern => /OK/, :timeout => 3.seconds
7
+
8
+ param :url, String, true
9
+ param :pattern, [String, Regexp]
10
+ param :kind, [String, Fixnum, Symbol]
11
+ param :timeout, [Fixnum, Float]
12
+ param :open_timeout, [Fixnum, Float]
13
+ param :read_timeout, [Fixnum, Float]
14
+
15
+ attr_reader :uri
16
+
17
+ def initialize(*args)
18
+ super
19
+
20
+ @uri = URI.parse(url)
21
+ @kind = case kind
22
+ when Fixnum then Net::HTTPResponse::CODE_TO_OBJ[kind]
23
+ when String, Symbol then Net.const_get("HTTP#{kind.to_s.camelize}") rescue Net::HTTPSuccess
24
+ else
25
+ Net::HTTPSuccess
26
+ end
27
+ @open_timeout = (open_timeout || 3).to_f
28
+ @read_timeout = (read_timeout || timeout || 15).to_f
29
+ end
30
+
31
+ def get_value
32
+ res = session.start{ |http| http.get(@uri.request_uri) }
33
+ {:result => res}
34
+
35
+ rescue Timeout::Error => ex
36
+ debug ex.inspect
37
+
38
+ if defined?(Net::OpenTimeout) # for ruby 2.0
39
+ mes = ex.class.is_a?(Net::OpenTimeout) ? "OpenTimeout<#{@open_timeout}>" : "ReadTimeout<#{@read_timeout}>"
40
+ {:exception => mes}
41
+ else
42
+ {:exception => "Timeout<#{@open_timeout},#{@read_timeout}>"}
43
+ end
44
+
45
+ rescue => ex
46
+ {:exception => "Error<#{ex.message}>"}
47
+ end
48
+
49
+ def good?(value)
50
+ return false unless value[:result]
51
+
52
+ unless value[:result].kind_of?(@kind)
53
+ return false
54
+ end
55
+
56
+ if pattern
57
+ matched = if pattern.is_a?(Regexp)
58
+ pattern === value[:result].body
59
+ else
60
+ value[:result].body.include?(pattern.to_s)
61
+ end
62
+ value[:notice] = "missing '#{pattern.to_s}'" unless matched
63
+ matched
64
+ else
65
+ true
66
+ end
67
+ end
68
+
69
+ def human_value(value)
70
+ if !value.is_a?(Hash)
71
+ '-'
72
+ elsif value[:exception]
73
+ value[:exception]
74
+ else
75
+ body_size = value[:result].body.size / 1024
76
+ msg = "#{value[:result].code}=#{body_size}Kb"
77
+ msg += "<#{value[:notice]}>" if value[:notice]
78
+ msg
79
+ end
80
+ end
81
+
82
+ private
83
+
84
+ def session
85
+ Net::HTTP.new(@uri.host, @uri.port).tap do |session|
86
+ if @uri.scheme == 'https'
87
+ require 'net/https'
88
+ session.use_ssl = true
89
+ session.verify_mode = OpenSSL::SSL::VERIFY_NONE
90
+ end
91
+ session.open_timeout = @open_timeout
92
+ session.read_timeout = @read_timeout
93
+ end
94
+ end
95
+
96
+ end
@@ -0,0 +1,17 @@
1
+ class Eye::Checker::Memory < Eye::Checker::Measure
2
+
3
+ # check :memory, :every => 3.seconds, :below => 80.megabytes, :times => [3,5]
4
+
5
+ def check_name
6
+ @check_name ||= "memory(#{measure_str})"
7
+ end
8
+
9
+ def get_value
10
+ Eye::SystemResources.memory(@pid).to_i
11
+ end
12
+
13
+ def human_value(value)
14
+ "#{value.to_i / 1024 / 1024}Mb"
15
+ end
16
+
17
+ end
@@ -0,0 +1,6 @@
1
+ class Eye::Checker::Nop < Eye::Checker
2
+
3
+ # check :nop, :every => 10.hours # means restart every 10 hours
4
+
5
+ def get_value; end
6
+ end
@@ -0,0 +1,18 @@
1
+ class Eye::Checker::Runtime < Eye::Checker::Measure
2
+
3
+ # check :runtime, :every => 1.minute, :below => 120.minutes
4
+
5
+ def get_value
6
+ st = Eye::SystemResources.start_time(@pid)
7
+ if st
8
+ Time.now.to_i - st.to_i
9
+ else
10
+ 0
11
+ end
12
+ end
13
+
14
+ def human_value(value)
15
+ "#{value / 60}m"
16
+ end
17
+
18
+ end
@@ -0,0 +1,159 @@
1
+ class Eye::Checker::Socket < Eye::Checker::Defer
2
+
3
+ # check :socket, :every => 5.seconds, :times => 1,
4
+ # :addr => "unix:/var/run/daemon.sock", :timeout => 3.seconds,
5
+ #
6
+ # Available parameters:
7
+ # :addr the socket addr to open. The format is tcp://<host>:<port> or unix:<path>
8
+ # :timeout generic timeout for reading data from socket
9
+ # :open_timeout override generic timeout for the connection
10
+ # :read_timeout override generic timeout for data read/write
11
+ # :send_data after connection send this data
12
+ # :expect_data after sending :send_data expect this response. Can be a string, Regexp or a Proc
13
+ # :protocol way of pack,unpack messages (default = socket default), example: :protocol => :em_object
14
+
15
+ param :addr, String, true
16
+ param :timeout, [Fixnum, Float]
17
+ param :open_timeout, [Fixnum, Float]
18
+ param :read_timeout, [Fixnum, Float]
19
+ param :send_data
20
+ param :expect_data, [String, Regexp, Proc]
21
+ param :protocol, [Symbol], nil, nil, [:default, :em_object, :raw]
22
+
23
+ def initialize(*args)
24
+ super
25
+ @open_timeout = (open_timeout || 1).to_f
26
+ @read_timeout = (read_timeout || timeout || 5).to_f
27
+
28
+ if addr =~ %r[\Atcp://(.*?):(.*?)\z]
29
+ @socket_family = :tcp
30
+ @socket_addr = $1
31
+ @socket_port = $2.to_i
32
+ elsif addr =~ %r[\Aunix:(.*)\z]
33
+ @socket_family = :unix
34
+ @socket_path = $1
35
+ end
36
+ end
37
+
38
+ def get_value
39
+ sock = begin
40
+ Timeout::timeout(@open_timeout){ open_socket }
41
+ rescue Timeout::Error
42
+ return { :exception => "OpenTimeout<#{@open_timeout}>" }
43
+ end
44
+
45
+ if send_data
46
+ begin
47
+ Timeout::timeout(@read_timeout) do
48
+ _write_data(sock, send_data)
49
+ result = _read_data(sock)
50
+
51
+ { :result => result }
52
+ end
53
+ rescue Timeout::Error
54
+ if protocol == :raw
55
+ return { :result => @buffer }
56
+ else
57
+ return { :exception => "ReadTimeout<#{@read_timeout}>" }
58
+ end
59
+ end
60
+ else
61
+ { :result => :listen }
62
+ end
63
+
64
+ rescue Exception => e
65
+ { :exception => "Error<#{e.message}>" }
66
+
67
+ ensure
68
+ sock.close if sock
69
+ end
70
+
71
+ def good?(value)
72
+ return false if !value[:result]
73
+
74
+ if expect_data
75
+ if expect_data.is_a?(Proc)
76
+ match = begin
77
+ !!expect_data[value[:result]]
78
+ rescue Timeout::Error, Exception => ex
79
+ mes = "proc match failed with '#{ex.message}'"
80
+ error(mes)
81
+ value[:notice] = mes
82
+ return false
83
+ end
84
+
85
+ unless match
86
+ warn "proc #{expect_data} not matched (#{value[:result].truncate(30)}) answer"
87
+ value[:notice] = 'missing proc validation'
88
+ end
89
+
90
+ return match
91
+ end
92
+
93
+ return true if expect_data.is_a?(Regexp) && expect_data.match(value[:result])
94
+ return true if value[:result].to_s == expect_data.to_s
95
+
96
+ warn "#{expect_data} not matched (#{value[:result].truncate(30)}) answer"
97
+ value[:notice] = "missing '#{expect_data.to_s}'"
98
+ return false
99
+ end
100
+
101
+ return true
102
+ end
103
+
104
+ def human_value(value)
105
+ if !value.is_a?(Hash)
106
+ '-'
107
+ elsif value[:exception]
108
+ value[:exception]
109
+ else
110
+ if value[:result] == :listen
111
+ 'listen'
112
+ else
113
+ res = "#{value[:result].to_s.size}b"
114
+ res += "<#{value[:notice]}>" if value[:notice]
115
+ res
116
+ end
117
+ end
118
+ end
119
+
120
+ private
121
+
122
+ def open_socket
123
+ if @socket_family == :tcp
124
+ TCPSocket.open(@socket_addr, @socket_port)
125
+ elsif @socket_family == :unix
126
+ UNIXSocket.open(@socket_path)
127
+ else
128
+ raise "Unknown socket addr #{addr}"
129
+ end
130
+ end
131
+
132
+ def _write_data(socket, data)
133
+ case protocol
134
+ when :em_object
135
+ data = Marshal.dump(data)
136
+ socket.write([data.bytesize, data].pack('Na*'))
137
+ else
138
+ socket.write(data.to_s)
139
+ end
140
+ end
141
+
142
+ def _read_data(socket)
143
+ case protocol
144
+ when :em_object
145
+ content = ''
146
+ msg_size = socket.recv(4).unpack('N')[0] rescue 0
147
+ content << socket.recv(msg_size - content.length) while content.length < msg_size
148
+ if content.present?
149
+ Marshal.load(content) rescue 'corrupted_marshal'
150
+ end
151
+ when :raw
152
+ @buffer = ''
153
+ loop { @buffer << socket.recv(1) }
154
+ else
155
+ socket.readline.chop
156
+ end
157
+ end
158
+
159
+ end