reel-eye 0.3.2 → 0.4

Sign up to get free protection for your applications and to get access to all the features.
Files changed (82) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +1 -0
  3. data/.rspec +1 -1
  4. data/.travis.yml +3 -1
  5. data/CHANGES.md +11 -2
  6. data/Gemfile +1 -0
  7. data/README.md +18 -14
  8. data/Rakefile +10 -3
  9. data/bin/eye +41 -27
  10. data/examples/process_thin.rb +1 -1
  11. data/examples/processes/em.rb +2 -2
  12. data/examples/processes/forking.rb +2 -2
  13. data/examples/processes/sample.rb +5 -5
  14. data/examples/rbenv.eye +1 -1
  15. data/examples/sidekiq.eye +2 -2
  16. data/examples/test.eye +10 -6
  17. data/examples/thin-farm.eye +1 -1
  18. data/examples/unicorn.eye +1 -1
  19. data/eye.gemspec +13 -7
  20. data/lib/eye.rb +6 -6
  21. data/lib/eye/application.rb +9 -6
  22. data/lib/eye/checker.rb +51 -21
  23. data/lib/eye/checker/file_size.rb +1 -1
  24. data/lib/eye/checker/http.rb +3 -3
  25. data/lib/eye/checker/memory.rb +1 -1
  26. data/lib/eye/checker/socket.rb +6 -6
  27. data/lib/eye/child_process.rb +7 -11
  28. data/lib/eye/client.rb +6 -6
  29. data/lib/eye/config.rb +2 -2
  30. data/lib/eye/controller.rb +13 -8
  31. data/lib/eye/controller/commands.rb +9 -10
  32. data/lib/eye/controller/helpers.rb +1 -0
  33. data/lib/eye/controller/load.rb +11 -7
  34. data/lib/eye/controller/options.rb +1 -1
  35. data/lib/eye/controller/send_command.rb +44 -19
  36. data/lib/eye/controller/show_history.rb +8 -7
  37. data/lib/eye/controller/status.rb +39 -26
  38. data/lib/eye/dsl.rb +3 -3
  39. data/lib/eye/dsl/application_opts.rb +4 -4
  40. data/lib/eye/dsl/config_opts.rb +4 -4
  41. data/lib/eye/dsl/helpers.rb +2 -2
  42. data/lib/eye/dsl/main.rb +2 -2
  43. data/lib/eye/dsl/opts.rb +19 -14
  44. data/lib/eye/dsl/process_opts.rb +1 -1
  45. data/lib/eye/dsl/pure_opts.rb +2 -2
  46. data/lib/eye/dsl/validation.rb +7 -5
  47. data/lib/eye/group.rb +17 -11
  48. data/lib/eye/group/chain.rb +3 -3
  49. data/lib/eye/http.rb +1 -1
  50. data/lib/eye/http/router.rb +2 -2
  51. data/lib/eye/loader.rb +8 -7
  52. data/lib/eye/logger.rb +14 -5
  53. data/lib/eye/notify.rb +13 -7
  54. data/lib/eye/notify/jabber.rb +2 -2
  55. data/lib/eye/notify/mail.rb +2 -2
  56. data/lib/eye/process.rb +10 -13
  57. data/lib/eye/process/child.rb +1 -1
  58. data/lib/eye/process/commands.rb +34 -32
  59. data/lib/eye/process/config.rb +17 -12
  60. data/lib/eye/process/controller.rb +3 -6
  61. data/lib/eye/process/data.rb +16 -5
  62. data/lib/eye/process/monitor.rb +12 -5
  63. data/lib/eye/process/notify.rb +1 -1
  64. data/lib/eye/process/scheduler.rb +3 -3
  65. data/lib/eye/process/states.rb +10 -13
  66. data/lib/eye/process/states_history.rb +3 -3
  67. data/lib/eye/process/system.rb +17 -21
  68. data/lib/eye/process/trigger.rb +11 -30
  69. data/lib/eye/process/watchers.rb +9 -9
  70. data/lib/eye/server.rb +14 -6
  71. data/lib/eye/settings.rb +4 -4
  72. data/lib/eye/system.rb +10 -7
  73. data/lib/eye/system_resources.rb +4 -4
  74. data/lib/eye/trigger.rb +58 -21
  75. data/lib/eye/trigger/flapping.rb +24 -4
  76. data/lib/eye/trigger/state.rb +28 -0
  77. data/lib/eye/utils/alive_array.rb +1 -1
  78. data/lib/eye/utils/celluloid_klass.rb +5 -0
  79. data/lib/eye/utils/pmap.rb +7 -0
  80. data/lib/eye/utils/tail.rb +1 -1
  81. metadata +252 -271
  82. data/lib/eye/utils/leak_19.rb +0 -7
@@ -13,7 +13,7 @@ module Eye::Process::Notify
13
13
 
14
14
  # send notifies
15
15
  if self[:notify].present?
16
- message = {:message => msg, :name => name,
16
+ message = {:message => msg, :name => name,
17
17
  :full_name => full_name, :pid => pid, :host => Eye::System.host, :level => level,
18
18
  :at => Time.now }
19
19
 
@@ -17,7 +17,7 @@ module Eye::Process::Scheduler
17
17
  if reason.class == Eye::Reason
18
18
  # for auto reasons
19
19
  # skip already running commands and all in chain
20
- scheduler.add_wo_dups_current(:scheduled_action, command, {:args => args, :reason => reason}, &block)
20
+ scheduler.add_wo_dups_current(:scheduled_action, command, {:args => args, :reason => reason}, &block)
21
21
  else
22
22
  # for manual, or without reason
23
23
  # skip only for last in chain
@@ -57,7 +57,7 @@ module Eye::Process::Scheduler
57
57
  def scheduler_clear_pending_list
58
58
  scheduler.clear_pending_list
59
59
  end
60
-
60
+
61
61
  def self.included(base)
62
62
  base.finalizer :remove_scheduler
63
63
  end
@@ -74,7 +74,7 @@ private
74
74
  def remove_scheduler
75
75
  @scheduler.terminate if @scheduler && @scheduler.alive?
76
76
  end
77
-
77
+
78
78
  def scheduler
79
79
  @scheduler ||= Eye::Utils::CelluloidChain.new(current_actor)
80
80
  end
@@ -1,4 +1,5 @@
1
1
  require 'state_machine'
2
+ require 'state_machine/version'
2
3
 
3
4
  class Eye::Process
4
5
 
@@ -52,11 +53,17 @@ class Eye::Process
52
53
  transition any => :unmonitored
53
54
  end
54
55
 
55
- after_transition any => :unmonitored, :do => :on_unmonitored
56
- after_transition any-:up => :up, :do => :on_up
57
- after_transition :up => any-:up, :do => :from_up
58
56
  after_transition any => any, :do => :log_transition
59
57
  after_transition any => any, :do => :check_triggers
58
+
59
+ after_transition any => :unmonitored, :do => :on_unmonitored
60
+
61
+ after_transition any-:up => :up, :do => :add_watchers
62
+ after_transition :up => any-:up, :do => :remove_watchers
63
+
64
+ after_transition any-:up => :up, :do => :add_childs
65
+ after_transition any => [:unmonitored, :down], :do => :remove_childs
66
+
60
67
  after_transition :on => :crashed, :do => :on_crashed
61
68
  end
62
69
 
@@ -68,16 +75,6 @@ class Eye::Process
68
75
  self.pid = nil
69
76
  end
70
77
 
71
- def on_up
72
- add_watchers
73
- add_childs
74
- end
75
-
76
- def from_up
77
- remove_watchers
78
- remove_childs
79
- end
80
-
81
78
  def log_transition(transition)
82
79
  @states_history.push transition.to_name, @state_reason
83
80
  info "switch :#{transition.event} [:#{transition.from_name} => :#{transition.to_name}] #{@state_reason ? "(reason: #{@state_reason})" : nil}"
@@ -40,19 +40,19 @@ class Eye::Process::StatesHistory < Eye::Utils::Tail
40
40
  end
41
41
 
42
42
  def any?(*seq)
43
- states.any? do |st|
43
+ states.any? do |st|
44
44
  seq.flatten.include?(st)
45
45
  end
46
46
  end
47
47
 
48
48
  def noone?(*seq)
49
- !states.all? do |st|
49
+ !states.all? do |st|
50
50
  seq.flatten.include?(st)
51
51
  end
52
52
  end
53
53
 
54
54
  def all?(*seq)
55
- states.all? do |st|
55
+ states.all? do |st|
56
56
  seq.flatten.include?(st)
57
57
  end
58
58
  end
@@ -2,9 +2,9 @@ require 'timeout'
2
2
 
3
3
  module Eye::Process::System
4
4
 
5
- def load_pid_from_file
5
+ def load_pid_from_file
6
6
  if File.exists?(self[:pid_file_ex])
7
- _pid = File.read(self[:pid_file_ex]).to_i
7
+ _pid = File.read(self[:pid_file_ex]).to_i
8
8
  _pid > 0 ? _pid : nil
9
9
  end
10
10
  end
@@ -25,9 +25,10 @@ module Eye::Process::System
25
25
  end
26
26
 
27
27
  def clear_pid_file
28
+ info "delete pid_file: #{self[:pid_file_ex]}"
28
29
  File.unlink(self[:pid_file_ex])
29
30
  true
30
- rescue
31
+ rescue
31
32
  nil
32
33
  end
33
34
 
@@ -51,9 +52,18 @@ module Eye::Process::System
51
52
  res[:result] == :ok
52
53
  end
53
54
 
54
- # non blocking actor timeout
55
55
  def wait_for_condition(timeout, step = 0.1, &block)
56
- defer{ wait_for_condition_sync(timeout, step, &block) }
56
+ res = nil
57
+ sumtime = 0
58
+
59
+ loop do
60
+ tm = Time.now
61
+ res = yield # note that yield can block actor here and timeout can be overhead
62
+ return res if res
63
+ sleep step.to_f
64
+ sumtime += (Time.now - tm)
65
+ return false if sumtime > timeout
66
+ end
57
67
  end
58
68
 
59
69
  def execute(cmd, cfg = {})
@@ -63,7 +73,7 @@ module Eye::Process::System
63
73
  def failsafe_load_pid
64
74
  pid = load_pid_from_file
65
75
 
66
- if !pid
76
+ if !pid
67
77
  # this is can be symlink changed case
68
78
  sleep 0.1
69
79
  pid = load_pid_from_file
@@ -73,25 +83,11 @@ module Eye::Process::System
73
83
  end
74
84
 
75
85
  def failsafe_save_pid
76
- save_pid_to_file
86
+ save_pid_to_file
77
87
  true
78
88
  rescue => ex
79
89
  error "failsafe_save_pid: #{ex.message}"
80
90
  false
81
91
  end
82
92
 
83
- private
84
-
85
- def wait_for_condition_sync(timeout, step, &block)
86
- res = nil
87
-
88
- Timeout::timeout(timeout.to_f) do
89
- sleep step.to_f until res = yield
90
- end
91
-
92
- res
93
- rescue Timeout::Error
94
- false
95
- end
96
-
97
93
  end
@@ -4,7 +4,7 @@ module Eye::Process::Trigger
4
4
  if self[:triggers]
5
5
  self[:triggers].each do |type, cfg|
6
6
  add_trigger(cfg)
7
- end
7
+ end
8
8
  end
9
9
  end
10
10
 
@@ -12,43 +12,24 @@ module Eye::Process::Trigger
12
12
  self.triggers = []
13
13
  end
14
14
 
15
- def check_triggers
15
+ def check_triggers(transition)
16
16
  return if unmonitored?
17
+ self.triggers.each { |trigger| trigger.notify(transition) }
18
+ end
17
19
 
18
- self.triggers.each do |trigger|
19
- unless trigger.check(self.states_history)
20
- on_flapping(trigger) if trigger.class == Eye::Trigger::Flapping
21
- end
22
- end
20
+ def retry_start_after_flapping
21
+ return unless unmonitored?
22
+ return unless state_reason.to_s.include?('flapping') # TODO: remove hackety
23
+
24
+ schedule :start, Eye::Reason.new(:'retry start after flapping')
25
+ self.flapping_times += 1
23
26
  end
24
27
 
25
28
  private
26
29
 
27
30
  def add_trigger(cfg = {})
28
- trigger = Eye::Trigger.create(cfg, logger.prefix)
31
+ trigger = Eye::Trigger.create(current_actor, cfg)
29
32
  self.triggers << trigger
30
33
  end
31
34
 
32
- def on_flapping(trigger)
33
- notify :error, 'flapping!'
34
- schedule :unmonitor, Eye::Reason.new(:flapping)
35
-
36
- @retry_times ||= 0
37
- retry_in = trigger.retry_in
38
-
39
- return unless retry_in
40
- return if trigger.retry_times && @retry_times >= trigger.retry_times
41
-
42
- schedule_in(retry_in.to_f, :retry_action)
43
- end
44
-
45
- def retry_action
46
- debug "trigger retry timer"
47
- return unless unmonitored?
48
- return unless state_reason.to_s.include?('flapping') # TODO: remove hackety
49
-
50
- schedule :start, Eye::Reason.new(:'retry start after flapping')
51
- @retry_times += 1
52
- end
53
-
54
35
  end
@@ -7,13 +7,13 @@ module Eye::Process::Watchers
7
7
 
8
8
  if @watchers.blank?
9
9
  # default watcher :check_alive
10
- add_watcher(:check_alive, self[:check_alive_period]) do
10
+ add_watcher(:check_alive, self[:check_alive_period]) do
11
11
  check_alive
12
12
  end
13
13
 
14
14
  # monitor childs pids
15
15
  if self[:monitor_children]
16
- add_watcher(:check_childs, self[:childs_update_period]) do
16
+ add_watcher(:check_childs, self[:childs_update_period]) do
17
17
  add_or_update_childs
18
18
  end
19
19
  end
@@ -24,13 +24,13 @@ module Eye::Process::Watchers
24
24
  warn 'try add_watchers, but its already here'
25
25
  end
26
26
  end
27
-
27
+
28
28
  def remove_watchers
29
29
  @watchers.each{|_, h| h[:timer].cancel }
30
30
  @watchers = {}
31
31
  end
32
32
 
33
- private
33
+ private
34
34
 
35
35
  def add_watcher(type, period = 2, subject = nil, &block)
36
36
  return if @watchers[type]
@@ -40,17 +40,17 @@ private
40
40
  timer = every(period.to_f) do
41
41
  debug "check #{type}"
42
42
  block.call(subject)
43
- end
43
+ end
44
44
 
45
45
  @watchers[type] ||= {:timer => timer, :subject => subject}
46
- end
47
-
46
+ end
47
+
48
48
  def start_checkers
49
49
  self[:checks].each{|name, cfg| start_checker(name, cfg) }
50
50
  end
51
51
 
52
52
  def start_checker(name, cfg)
53
- subject = Eye::Checker.create(pid, cfg, logger.prefix)
53
+ subject = Eye::Checker.create(pid, cfg, current_actor)
54
54
 
55
55
  # ex: {:type => :memory, :every => 5.seconds, :below => 100.megabytes, :times => [3,5]}
56
56
  add_watcher("check_#{name}".to_sym, subject.every, subject, &method(:watcher_tick).to_proc)
@@ -59,7 +59,7 @@ private
59
59
  def watcher_tick(subject)
60
60
  unless subject.check
61
61
  return unless up?
62
-
62
+
63
63
  action = subject.fire || :restart
64
64
  notify :warn, "Bounded #{subject.check_name}: #{subject.last_human_values} send to :#{action}"
65
65
  schedule action, Eye::Reason.new("bounded #{subject.check_name}")
data/lib/eye/server.rb CHANGED
@@ -3,9 +3,9 @@ require 'celluloid/autostart'
3
3
 
4
4
  class Eye::Server
5
5
  include Celluloid::IO
6
-
6
+
7
7
  attr_reader :socket_path, :server
8
-
8
+
9
9
  def initialize(socket_path)
10
10
  @socket_path = socket_path
11
11
  @server = begin
@@ -21,14 +21,22 @@ class Eye::Server
21
21
  end
22
22
 
23
23
  def handle_connection(socket)
24
- command, *args = socket.readline.strip.split('|')
24
+ text = socket.read
25
+
26
+ begin
27
+ command, *args = Marshal.load(text)
28
+ rescue => ex
29
+ error "Failed socket read #{ex.message}"
30
+ return
31
+ end
32
+
25
33
  response = command(command, *args)
26
34
  socket.write(Marshal.dump(response))
27
-
35
+
28
36
  rescue Errno::EPIPE
29
37
  # client timeouted
30
38
  # do nothing
31
-
39
+
32
40
  ensure
33
41
  socket.close
34
42
  end
@@ -43,7 +51,7 @@ class Eye::Server
43
51
  end
44
52
 
45
53
  finalizer :close_socket
46
-
54
+
47
55
  def close_socket
48
56
  @server.close if @server
49
57
  unlink_socket_file
data/lib/eye/settings.rb CHANGED
@@ -2,7 +2,7 @@ require 'fileutils'
2
2
 
3
3
  module Eye::Settings
4
4
  module_function
5
-
5
+
6
6
  def dir
7
7
  if root?
8
8
  '/var/run/eye'
@@ -26,7 +26,7 @@ module Eye::Settings
26
26
  def home
27
27
  ENV['EYE_HOME'] || ENV['HOME']
28
28
  end
29
-
29
+
30
30
  def path(path)
31
31
  File.join(dir, path)
32
32
  end
@@ -38,11 +38,11 @@ module Eye::Settings
38
38
  def socket_path
39
39
  path(ENV['EYE_SOCK'] || "sock#{ENV['EYE_V']}")
40
40
  end
41
-
41
+
42
42
  def pid_path
43
43
  path(ENV['EYE_PID'] || "pid#{ENV['EYE_V']}")
44
44
  end
45
-
45
+
46
46
  def cache_path
47
47
  path("processes#{ENV['EYE_V']}.cache")
48
48
  end
data/lib/eye/system.rb CHANGED
@@ -29,7 +29,7 @@ module Eye::System
29
29
 
30
30
  # Send signal to process (uses for kill)
31
31
  # code: TERM(15), KILL(9), QUIT(3), ...
32
- def send_signal(pid, code = :TERM)
32
+ def send_signal(pid, code = :TERM)
33
33
  code = 0 if code == '0'
34
34
  if code.to_s.to_i != 0
35
35
  code = code.to_i
@@ -38,7 +38,7 @@ module Eye::System
38
38
  code = code.to_s.upcase if code.is_a?(String) || code.is_a?(Symbol)
39
39
 
40
40
  if pid
41
- ::Process.kill(code, pid)
41
+ ::Process.kill(code, pid)
42
42
  {:result => :ok}
43
43
  else
44
44
  {:error => Exception.new('no_pid')}
@@ -59,7 +59,7 @@ module Eye::System
59
59
  pid = Process::spawn(prepare_env(cfg), *Shellwords.shellwords(cmd), opts)
60
60
  Process.detach(pid)
61
61
  {:pid => pid}
62
-
62
+
63
63
  rescue Errno::ENOENT, Errno::EACCES => ex
64
64
  {:error => ex}
65
65
  end
@@ -82,7 +82,7 @@ module Eye::System
82
82
 
83
83
  rescue Timeout::Error => ex
84
84
  if pid
85
- Eye.warn "[#{cfg[:name]}] send signal 9 to #{pid} (because of timeouted<#{timeout}> execution)"
85
+ warn "[#{cfg[:name]}] send signal 9 to #{pid} (because of timeouted<#{timeout}> execution)"
86
86
  send_signal(pid, 9)
87
87
  end
88
88
  {:error => ex}
@@ -90,7 +90,7 @@ module Eye::System
90
90
  rescue Errno::ENOENT, Errno::EACCES => ex
91
91
  {:error => ex}
92
92
 
93
- ensure
93
+ ensure
94
94
  Process.detach(pid) if pid
95
95
  end
96
96
 
@@ -102,7 +102,7 @@ module Eye::System
102
102
  h = {}
103
103
  str.each_line do |line|
104
104
  chunk = line.strip.split(/\s+/)
105
- h[chunk[0].to_i] = { :ppid => chunk[1].to_i, :cpu => chunk[2].to_i,
105
+ h[chunk[0].to_i] = { :ppid => chunk[1].to_i, :cpu => chunk[2].to_i,
106
106
  :rss => chunk[3].to_i, :start_time => chunk[4] }
107
107
  end
108
108
  h
@@ -114,7 +114,10 @@ module Eye::System
114
114
  end
115
115
 
116
116
  def host
117
- @host ||= `hostname`.chomp
117
+ @host ||= begin
118
+ require 'socket'
119
+ Socket.gethostname
120
+ end
118
121
  end
119
122
 
120
123
  # set host for tests