eye 0.3.2 → 0.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (79) hide show
  1. checksums.yaml +4 -4
  2. data/.gitignore +1 -0
  3. data/.rspec +1 -1
  4. data/.travis.yml +3 -1
  5. data/CHANGES.md +11 -2
  6. data/Gemfile +1 -0
  7. data/README.md +18 -14
  8. data/Rakefile +10 -3
  9. data/bin/eye +41 -27
  10. data/examples/process_thin.rb +1 -1
  11. data/examples/processes/em.rb +2 -2
  12. data/examples/processes/forking.rb +2 -2
  13. data/examples/processes/sample.rb +5 -5
  14. data/examples/rbenv.eye +1 -1
  15. data/examples/sidekiq.eye +2 -2
  16. data/examples/test.eye +10 -6
  17. data/examples/thin-farm.eye +1 -1
  18. data/examples/unicorn.eye +1 -1
  19. data/eye.gemspec +13 -7
  20. data/lib/eye.rb +6 -6
  21. data/lib/eye/application.rb +9 -6
  22. data/lib/eye/checker.rb +51 -21
  23. data/lib/eye/checker/file_size.rb +1 -1
  24. data/lib/eye/checker/http.rb +3 -3
  25. data/lib/eye/checker/memory.rb +1 -1
  26. data/lib/eye/checker/socket.rb +6 -6
  27. data/lib/eye/child_process.rb +7 -11
  28. data/lib/eye/client.rb +6 -6
  29. data/lib/eye/config.rb +2 -2
  30. data/lib/eye/controller.rb +11 -8
  31. data/lib/eye/controller/commands.rb +8 -9
  32. data/lib/eye/controller/helpers.rb +1 -0
  33. data/lib/eye/controller/load.rb +11 -7
  34. data/lib/eye/controller/send_command.rb +44 -19
  35. data/lib/eye/controller/show_history.rb +8 -7
  36. data/lib/eye/controller/status.rb +39 -26
  37. data/lib/eye/dsl.rb +3 -3
  38. data/lib/eye/dsl/application_opts.rb +4 -4
  39. data/lib/eye/dsl/config_opts.rb +4 -4
  40. data/lib/eye/dsl/helpers.rb +2 -2
  41. data/lib/eye/dsl/main.rb +2 -2
  42. data/lib/eye/dsl/opts.rb +19 -14
  43. data/lib/eye/dsl/process_opts.rb +1 -1
  44. data/lib/eye/dsl/pure_opts.rb +2 -2
  45. data/lib/eye/dsl/validation.rb +7 -5
  46. data/lib/eye/group.rb +17 -11
  47. data/lib/eye/group/chain.rb +3 -3
  48. data/lib/eye/loader.rb +8 -6
  49. data/lib/eye/logger.rb +14 -5
  50. data/lib/eye/notify.rb +13 -7
  51. data/lib/eye/notify/jabber.rb +2 -2
  52. data/lib/eye/notify/mail.rb +2 -2
  53. data/lib/eye/process.rb +10 -13
  54. data/lib/eye/process/child.rb +1 -1
  55. data/lib/eye/process/commands.rb +34 -32
  56. data/lib/eye/process/config.rb +17 -12
  57. data/lib/eye/process/controller.rb +3 -6
  58. data/lib/eye/process/data.rb +16 -5
  59. data/lib/eye/process/monitor.rb +12 -5
  60. data/lib/eye/process/notify.rb +1 -1
  61. data/lib/eye/process/scheduler.rb +3 -3
  62. data/lib/eye/process/states.rb +10 -13
  63. data/lib/eye/process/states_history.rb +3 -3
  64. data/lib/eye/process/system.rb +17 -21
  65. data/lib/eye/process/trigger.rb +11 -30
  66. data/lib/eye/process/watchers.rb +9 -9
  67. data/lib/eye/server.rb +14 -6
  68. data/lib/eye/settings.rb +4 -4
  69. data/lib/eye/system.rb +10 -7
  70. data/lib/eye/system_resources.rb +4 -4
  71. data/lib/eye/trigger.rb +58 -21
  72. data/lib/eye/trigger/flapping.rb +24 -4
  73. data/lib/eye/trigger/state.rb +28 -0
  74. data/lib/eye/utils/alive_array.rb +1 -1
  75. data/lib/eye/utils/celluloid_klass.rb +5 -0
  76. data/lib/eye/utils/pmap.rb +7 -0
  77. data/lib/eye/utils/tail.rb +1 -1
  78. metadata +39 -23
  79. data/lib/eye/utils/leak_19.rb +0 -7
@@ -13,7 +13,7 @@ module Eye::Process::Notify
13
13
 
14
14
  # send notifies
15
15
  if self[:notify].present?
16
- message = {:message => msg, :name => name,
16
+ message = {:message => msg, :name => name,
17
17
  :full_name => full_name, :pid => pid, :host => Eye::System.host, :level => level,
18
18
  :at => Time.now }
19
19
 
@@ -17,7 +17,7 @@ module Eye::Process::Scheduler
17
17
  if reason.class == Eye::Reason
18
18
  # for auto reasons
19
19
  # skip already running commands and all in chain
20
- scheduler.add_wo_dups_current(:scheduled_action, command, {:args => args, :reason => reason}, &block)
20
+ scheduler.add_wo_dups_current(:scheduled_action, command, {:args => args, :reason => reason}, &block)
21
21
  else
22
22
  # for manual, or without reason
23
23
  # skip only for last in chain
@@ -57,7 +57,7 @@ module Eye::Process::Scheduler
57
57
  def scheduler_clear_pending_list
58
58
  scheduler.clear_pending_list
59
59
  end
60
-
60
+
61
61
  def self.included(base)
62
62
  base.finalizer :remove_scheduler
63
63
  end
@@ -74,7 +74,7 @@ private
74
74
  def remove_scheduler
75
75
  @scheduler.terminate if @scheduler && @scheduler.alive?
76
76
  end
77
-
77
+
78
78
  def scheduler
79
79
  @scheduler ||= Eye::Utils::CelluloidChain.new(current_actor)
80
80
  end
@@ -1,4 +1,5 @@
1
1
  require 'state_machine'
2
+ require 'state_machine/version'
2
3
 
3
4
  class Eye::Process
4
5
 
@@ -52,11 +53,17 @@ class Eye::Process
52
53
  transition any => :unmonitored
53
54
  end
54
55
 
55
- after_transition any => :unmonitored, :do => :on_unmonitored
56
- after_transition any-:up => :up, :do => :on_up
57
- after_transition :up => any-:up, :do => :from_up
58
56
  after_transition any => any, :do => :log_transition
59
57
  after_transition any => any, :do => :check_triggers
58
+
59
+ after_transition any => :unmonitored, :do => :on_unmonitored
60
+
61
+ after_transition any-:up => :up, :do => :add_watchers
62
+ after_transition :up => any-:up, :do => :remove_watchers
63
+
64
+ after_transition any-:up => :up, :do => :add_childs
65
+ after_transition any => [:unmonitored, :down], :do => :remove_childs
66
+
60
67
  after_transition :on => :crashed, :do => :on_crashed
61
68
  end
62
69
 
@@ -68,16 +75,6 @@ class Eye::Process
68
75
  self.pid = nil
69
76
  end
70
77
 
71
- def on_up
72
- add_watchers
73
- add_childs
74
- end
75
-
76
- def from_up
77
- remove_watchers
78
- remove_childs
79
- end
80
-
81
78
  def log_transition(transition)
82
79
  @states_history.push transition.to_name, @state_reason
83
80
  info "switch :#{transition.event} [:#{transition.from_name} => :#{transition.to_name}] #{@state_reason ? "(reason: #{@state_reason})" : nil}"
@@ -40,19 +40,19 @@ class Eye::Process::StatesHistory < Eye::Utils::Tail
40
40
  end
41
41
 
42
42
  def any?(*seq)
43
- states.any? do |st|
43
+ states.any? do |st|
44
44
  seq.flatten.include?(st)
45
45
  end
46
46
  end
47
47
 
48
48
  def noone?(*seq)
49
- !states.all? do |st|
49
+ !states.all? do |st|
50
50
  seq.flatten.include?(st)
51
51
  end
52
52
  end
53
53
 
54
54
  def all?(*seq)
55
- states.all? do |st|
55
+ states.all? do |st|
56
56
  seq.flatten.include?(st)
57
57
  end
58
58
  end
@@ -2,9 +2,9 @@ require 'timeout'
2
2
 
3
3
  module Eye::Process::System
4
4
 
5
- def load_pid_from_file
5
+ def load_pid_from_file
6
6
  if File.exists?(self[:pid_file_ex])
7
- _pid = File.read(self[:pid_file_ex]).to_i
7
+ _pid = File.read(self[:pid_file_ex]).to_i
8
8
  _pid > 0 ? _pid : nil
9
9
  end
10
10
  end
@@ -25,9 +25,10 @@ module Eye::Process::System
25
25
  end
26
26
 
27
27
  def clear_pid_file
28
+ info "delete pid_file: #{self[:pid_file_ex]}"
28
29
  File.unlink(self[:pid_file_ex])
29
30
  true
30
- rescue
31
+ rescue
31
32
  nil
32
33
  end
33
34
 
@@ -51,9 +52,18 @@ module Eye::Process::System
51
52
  res[:result] == :ok
52
53
  end
53
54
 
54
- # non blocking actor timeout
55
55
  def wait_for_condition(timeout, step = 0.1, &block)
56
- defer{ wait_for_condition_sync(timeout, step, &block) }
56
+ res = nil
57
+ sumtime = 0
58
+
59
+ loop do
60
+ tm = Time.now
61
+ res = yield # note that yield can block actor here and timeout can be overhead
62
+ return res if res
63
+ sleep step.to_f
64
+ sumtime += (Time.now - tm)
65
+ return false if sumtime > timeout
66
+ end
57
67
  end
58
68
 
59
69
  def execute(cmd, cfg = {})
@@ -63,7 +73,7 @@ module Eye::Process::System
63
73
  def failsafe_load_pid
64
74
  pid = load_pid_from_file
65
75
 
66
- if !pid
76
+ if !pid
67
77
  # this is can be symlink changed case
68
78
  sleep 0.1
69
79
  pid = load_pid_from_file
@@ -73,25 +83,11 @@ module Eye::Process::System
73
83
  end
74
84
 
75
85
  def failsafe_save_pid
76
- save_pid_to_file
86
+ save_pid_to_file
77
87
  true
78
88
  rescue => ex
79
89
  error "failsafe_save_pid: #{ex.message}"
80
90
  false
81
91
  end
82
92
 
83
- private
84
-
85
- def wait_for_condition_sync(timeout, step, &block)
86
- res = nil
87
-
88
- Timeout::timeout(timeout.to_f) do
89
- sleep step.to_f until res = yield
90
- end
91
-
92
- res
93
- rescue Timeout::Error
94
- false
95
- end
96
-
97
93
  end
@@ -4,7 +4,7 @@ module Eye::Process::Trigger
4
4
  if self[:triggers]
5
5
  self[:triggers].each do |type, cfg|
6
6
  add_trigger(cfg)
7
- end
7
+ end
8
8
  end
9
9
  end
10
10
 
@@ -12,43 +12,24 @@ module Eye::Process::Trigger
12
12
  self.triggers = []
13
13
  end
14
14
 
15
- def check_triggers
15
+ def check_triggers(transition)
16
16
  return if unmonitored?
17
+ self.triggers.each { |trigger| trigger.notify(transition) }
18
+ end
17
19
 
18
- self.triggers.each do |trigger|
19
- unless trigger.check(self.states_history)
20
- on_flapping(trigger) if trigger.class == Eye::Trigger::Flapping
21
- end
22
- end
20
+ def retry_start_after_flapping
21
+ return unless unmonitored?
22
+ return unless state_reason.to_s.include?('flapping') # TODO: remove hackety
23
+
24
+ schedule :start, Eye::Reason.new(:'retry start after flapping')
25
+ self.flapping_times += 1
23
26
  end
24
27
 
25
28
  private
26
29
 
27
30
  def add_trigger(cfg = {})
28
- trigger = Eye::Trigger.create(cfg, logger.prefix)
31
+ trigger = Eye::Trigger.create(current_actor, cfg)
29
32
  self.triggers << trigger
30
33
  end
31
34
 
32
- def on_flapping(trigger)
33
- notify :error, 'flapping!'
34
- schedule :unmonitor, Eye::Reason.new(:flapping)
35
-
36
- @retry_times ||= 0
37
- retry_in = trigger.retry_in
38
-
39
- return unless retry_in
40
- return if trigger.retry_times && @retry_times >= trigger.retry_times
41
-
42
- schedule_in(retry_in.to_f, :retry_action)
43
- end
44
-
45
- def retry_action
46
- debug "trigger retry timer"
47
- return unless unmonitored?
48
- return unless state_reason.to_s.include?('flapping') # TODO: remove hackety
49
-
50
- schedule :start, Eye::Reason.new(:'retry start after flapping')
51
- @retry_times += 1
52
- end
53
-
54
35
  end
@@ -7,13 +7,13 @@ module Eye::Process::Watchers
7
7
 
8
8
  if @watchers.blank?
9
9
  # default watcher :check_alive
10
- add_watcher(:check_alive, self[:check_alive_period]) do
10
+ add_watcher(:check_alive, self[:check_alive_period]) do
11
11
  check_alive
12
12
  end
13
13
 
14
14
  # monitor childs pids
15
15
  if self[:monitor_children]
16
- add_watcher(:check_childs, self[:childs_update_period]) do
16
+ add_watcher(:check_childs, self[:childs_update_period]) do
17
17
  add_or_update_childs
18
18
  end
19
19
  end
@@ -24,13 +24,13 @@ module Eye::Process::Watchers
24
24
  warn 'try add_watchers, but its already here'
25
25
  end
26
26
  end
27
-
27
+
28
28
  def remove_watchers
29
29
  @watchers.each{|_, h| h[:timer].cancel }
30
30
  @watchers = {}
31
31
  end
32
32
 
33
- private
33
+ private
34
34
 
35
35
  def add_watcher(type, period = 2, subject = nil, &block)
36
36
  return if @watchers[type]
@@ -40,17 +40,17 @@ private
40
40
  timer = every(period.to_f) do
41
41
  debug "check #{type}"
42
42
  block.call(subject)
43
- end
43
+ end
44
44
 
45
45
  @watchers[type] ||= {:timer => timer, :subject => subject}
46
- end
47
-
46
+ end
47
+
48
48
  def start_checkers
49
49
  self[:checks].each{|name, cfg| start_checker(name, cfg) }
50
50
  end
51
51
 
52
52
  def start_checker(name, cfg)
53
- subject = Eye::Checker.create(pid, cfg, logger.prefix)
53
+ subject = Eye::Checker.create(pid, cfg, current_actor)
54
54
 
55
55
  # ex: {:type => :memory, :every => 5.seconds, :below => 100.megabytes, :times => [3,5]}
56
56
  add_watcher("check_#{name}".to_sym, subject.every, subject, &method(:watcher_tick).to_proc)
@@ -59,7 +59,7 @@ private
59
59
  def watcher_tick(subject)
60
60
  unless subject.check
61
61
  return unless up?
62
-
62
+
63
63
  action = subject.fire || :restart
64
64
  notify :warn, "Bounded #{subject.check_name}: #{subject.last_human_values} send to :#{action}"
65
65
  schedule action, Eye::Reason.new("bounded #{subject.check_name}")
@@ -3,9 +3,9 @@ require 'celluloid/autostart'
3
3
 
4
4
  class Eye::Server
5
5
  include Celluloid::IO
6
-
6
+
7
7
  attr_reader :socket_path, :server
8
-
8
+
9
9
  def initialize(socket_path)
10
10
  @socket_path = socket_path
11
11
  @server = begin
@@ -21,14 +21,22 @@ class Eye::Server
21
21
  end
22
22
 
23
23
  def handle_connection(socket)
24
- command, *args = socket.readline.strip.split('|')
24
+ text = socket.read
25
+
26
+ begin
27
+ command, *args = Marshal.load(text)
28
+ rescue => ex
29
+ error "Failed socket read #{ex.message}"
30
+ return
31
+ end
32
+
25
33
  response = command(command, *args)
26
34
  socket.write(Marshal.dump(response))
27
-
35
+
28
36
  rescue Errno::EPIPE
29
37
  # client timeouted
30
38
  # do nothing
31
-
39
+
32
40
  ensure
33
41
  socket.close
34
42
  end
@@ -43,7 +51,7 @@ class Eye::Server
43
51
  end
44
52
 
45
53
  finalizer :close_socket
46
-
54
+
47
55
  def close_socket
48
56
  @server.close if @server
49
57
  unlink_socket_file
@@ -2,7 +2,7 @@ require 'fileutils'
2
2
 
3
3
  module Eye::Settings
4
4
  module_function
5
-
5
+
6
6
  def dir
7
7
  if root?
8
8
  '/var/run/eye'
@@ -26,7 +26,7 @@ module Eye::Settings
26
26
  def home
27
27
  ENV['EYE_HOME'] || ENV['HOME']
28
28
  end
29
-
29
+
30
30
  def path(path)
31
31
  File.join(dir, path)
32
32
  end
@@ -38,11 +38,11 @@ module Eye::Settings
38
38
  def socket_path
39
39
  path(ENV['EYE_SOCK'] || "sock#{ENV['EYE_V']}")
40
40
  end
41
-
41
+
42
42
  def pid_path
43
43
  path(ENV['EYE_PID'] || "pid#{ENV['EYE_V']}")
44
44
  end
45
-
45
+
46
46
  def cache_path
47
47
  path("processes#{ENV['EYE_V']}.cache")
48
48
  end
@@ -29,7 +29,7 @@ module Eye::System
29
29
 
30
30
  # Send signal to process (uses for kill)
31
31
  # code: TERM(15), KILL(9), QUIT(3), ...
32
- def send_signal(pid, code = :TERM)
32
+ def send_signal(pid, code = :TERM)
33
33
  code = 0 if code == '0'
34
34
  if code.to_s.to_i != 0
35
35
  code = code.to_i
@@ -38,7 +38,7 @@ module Eye::System
38
38
  code = code.to_s.upcase if code.is_a?(String) || code.is_a?(Symbol)
39
39
 
40
40
  if pid
41
- ::Process.kill(code, pid)
41
+ ::Process.kill(code, pid)
42
42
  {:result => :ok}
43
43
  else
44
44
  {:error => Exception.new('no_pid')}
@@ -59,7 +59,7 @@ module Eye::System
59
59
  pid = Process::spawn(prepare_env(cfg), *Shellwords.shellwords(cmd), opts)
60
60
  Process.detach(pid)
61
61
  {:pid => pid}
62
-
62
+
63
63
  rescue Errno::ENOENT, Errno::EACCES => ex
64
64
  {:error => ex}
65
65
  end
@@ -82,7 +82,7 @@ module Eye::System
82
82
 
83
83
  rescue Timeout::Error => ex
84
84
  if pid
85
- Eye.warn "[#{cfg[:name]}] send signal 9 to #{pid} (because of timeouted<#{timeout}> execution)"
85
+ warn "[#{cfg[:name]}] send signal 9 to #{pid} (because of timeouted<#{timeout}> execution)"
86
86
  send_signal(pid, 9)
87
87
  end
88
88
  {:error => ex}
@@ -90,7 +90,7 @@ module Eye::System
90
90
  rescue Errno::ENOENT, Errno::EACCES => ex
91
91
  {:error => ex}
92
92
 
93
- ensure
93
+ ensure
94
94
  Process.detach(pid) if pid
95
95
  end
96
96
 
@@ -102,7 +102,7 @@ module Eye::System
102
102
  h = {}
103
103
  str.each_line do |line|
104
104
  chunk = line.strip.split(/\s+/)
105
- h[chunk[0].to_i] = { :ppid => chunk[1].to_i, :cpu => chunk[2].to_i,
105
+ h[chunk[0].to_i] = { :ppid => chunk[1].to_i, :cpu => chunk[2].to_i,
106
106
  :rss => chunk[3].to_i, :start_time => chunk[4] }
107
107
  end
108
108
  h
@@ -114,7 +114,10 @@ module Eye::System
114
114
  end
115
115
 
116
116
  def host
117
- @host ||= `hostname`.chomp
117
+ @host ||= begin
118
+ require 'socket'
119
+ Socket.gethostname
120
+ end
118
121
  end
119
122
 
120
123
  # set host for tests