eye 0.2.1 → 0.2.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (53) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +24 -17
  3. data/bin/eye +5 -0
  4. data/examples/notify.eye +18 -0
  5. data/examples/process_thin.rb +29 -0
  6. data/examples/sidekiq.eye +5 -1
  7. data/examples/test.eye +24 -17
  8. data/examples/thin-farm.eye +29 -0
  9. data/examples/unicorn.eye +2 -0
  10. data/eye.gemspec +1 -1
  11. data/lib/eye.rb +1 -1
  12. data/lib/eye/checker.rb +3 -5
  13. data/lib/eye/checker/http.rb +1 -2
  14. data/lib/eye/checker/socket.rb +2 -2
  15. data/lib/eye/controller/commands.rb +2 -0
  16. data/lib/eye/controller/send_command.rb +8 -2
  17. data/lib/eye/dsl.rb +1 -0
  18. data/lib/eye/dsl/opts.rb +36 -2
  19. data/lib/eye/dsl/pure_opts.rb +17 -39
  20. data/lib/eye/{checker → dsl}/validation.rb +15 -5
  21. data/lib/eye/group.rb +13 -4
  22. data/lib/eye/group/chain.rb +11 -0
  23. data/lib/eye/notify.rb +1 -1
  24. data/lib/eye/notify/mail.rb +1 -1
  25. data/lib/eye/process/child.rb +0 -2
  26. data/lib/eye/process/commands.rb +11 -14
  27. data/lib/eye/process/monitor.rb +10 -16
  28. data/lib/eye/process/scheduler.rb +8 -3
  29. data/lib/eye/process/states.rb +1 -2
  30. data/lib/eye/process/system.rb +38 -7
  31. data/lib/eye/process/watchers.rb +3 -3
  32. data/lib/eye/settings.rb +1 -1
  33. data/lib/eye/system.rb +33 -16
  34. data/lib/eye/trigger.rb +1 -1
  35. data/lib/eye/utils/celluloid_chain.rb +2 -0
  36. data/spec/checker/cpu_spec.rb +1 -1
  37. data/spec/checker/http_spec.rb +2 -2
  38. data/spec/checker/memory_spec.rb +2 -2
  39. data/spec/checker_spec.rb +1 -1
  40. data/spec/controller/controller_spec.rb +6 -0
  41. data/spec/controller/find_objects_spec.rb +6 -0
  42. data/spec/controller/intergration_spec.rb +24 -0
  43. data/spec/dsl/checks_spec.rb +2 -2
  44. data/spec/dsl/notify_spec.rb +12 -3
  45. data/spec/dsl/with_server_spec.rb +26 -2
  46. data/spec/process/checks/child_checks_spec.rb +1 -1
  47. data/spec/process/checks/memory_spec.rb +14 -0
  48. data/spec/process/scheduler_spec.rb +8 -0
  49. data/spec/process/system_spec.rb +27 -4
  50. data/spec/spec_helper.rb +3 -1
  51. data/spec/system_spec.rb +12 -5
  52. data/spec/utils/celluloid_chain_spec.rb +8 -0
  53. metadata +8 -5
@@ -66,49 +66,10 @@ class Eye::Dsl::PureOpts
66
66
  []
67
67
  end
68
68
 
69
- # execute part of config on particular server
70
- # array of strings
71
- # regexp
72
- # string
73
- def with_server(glob = nil, &block)
74
- on_server = true
75
-
76
- if glob.present?
77
- host = Eye::System.host
78
-
79
- if glob.is_a?(Array)
80
- on_server = !!glob.any?{|elem| elem == host}
81
- elsif glob.is_a?(Regexp)
82
- on_server = !!host.match(glob)
83
- elsif glob.is_a?(String) || glob.is_a?(Symbol)
84
- on_server = (host == glob.to_s)
85
- end
86
- end
87
-
88
- with_condition(on_server, &block)
89
-
90
- on_server
91
- end
92
-
93
69
  def with_condition(cond = true, &block)
94
70
  self.instance_eval(&block) if cond && block
95
71
  end
96
72
 
97
- def self.with_parsed_file(file_name)
98
- saved_parsed_filename = Eye.parsed_filename
99
-
100
- require 'pathname'
101
-
102
- real_filename = Eye.parsed_filename && File.symlink?(Eye.parsed_filename) ? File.readlink(Eye.parsed_filename) : Eye.parsed_filename
103
- dirname = File.dirname(real_filename) rescue nil
104
- path = Pathname.new(file_name).expand_path(dirname).to_s
105
-
106
- Eye.parsed_filename = path
107
- yield path
108
- ensure
109
- Eye.parsed_filename = saved_parsed_filename
110
- end
111
-
112
73
  def include(proc, *args)
113
74
  if proc.is_a?(String)
114
75
  self.class.with_parsed_file(proc) do |path|
@@ -129,4 +90,21 @@ class Eye::Dsl::PureOpts
129
90
  end
130
91
  end
131
92
 
93
+ private
94
+
95
+ def self.with_parsed_file(file_name)
96
+ saved_parsed_filename = Eye.parsed_filename
97
+
98
+ require 'pathname'
99
+
100
+ real_filename = Eye.parsed_filename && File.symlink?(Eye.parsed_filename) ? File.readlink(Eye.parsed_filename) : Eye.parsed_filename
101
+ dirname = File.dirname(real_filename) rescue nil
102
+ path = Pathname.new(file_name).expand_path(dirname).to_s
103
+
104
+ Eye.parsed_filename = path
105
+ yield path
106
+ ensure
107
+ Eye.parsed_filename = saved_parsed_filename
108
+ end
109
+
132
110
  end
@@ -1,24 +1,27 @@
1
- module Eye::Checker::Validation
1
+ module Eye::Dsl::Validation
2
2
  class Error < Exception; end
3
3
 
4
4
  def inherited(subclass)
5
5
  subclass.validates = self.validates.clone
6
6
  subclass.should_bes = self.should_bes.clone
7
7
  subclass.defaults = self.defaults.clone
8
+ subclass.variants = self.variants.clone
8
9
  end
9
10
 
10
- attr_accessor :validates, :should_bes, :defaults
11
+ attr_accessor :validates, :should_bes, :defaults, :variants
11
12
 
12
13
  def validates; @validates ||= {}; end
13
14
  def should_bes; @should_bes ||= []; end
14
15
  def defaults; @defaults ||= {}; end
16
+ def variants; @variants ||= {}; end
15
17
 
16
- def param(param, types = [], should_be = false, default = nil)
18
+ def param(param, types = [], should_be = false, default = nil, _variants = nil)
17
19
  param = param.to_sym
18
20
 
19
21
  validates[param] = types
20
22
  should_bes << param if should_be
21
23
  defaults[param] = default
24
+ variants[param] = _variants
22
25
 
23
26
  define_method "#{param}" do
24
27
  @options[param.to_sym] || default
@@ -27,13 +30,20 @@ module Eye::Checker::Validation
27
30
 
28
31
  def validate(options = {})
29
32
  options.each do |param, value|
30
- types = validates[param.to_sym]
33
+ param = param.to_sym
34
+ types = validates[param]
31
35
  unless types
32
- if param.to_sym != :type
36
+ if param != :type
33
37
  raise Error, "#{self.name} unknown param :#{param} value #{value.inspect}"
34
38
  end
35
39
  end
36
40
 
41
+ if self.variants[param]
42
+ if value && !value.is_a?(Proc) && !self.variants[param].include?(value)
43
+ raise Error, "#{value.inspect} should within #{self.variants[param].inspect}"
44
+ end
45
+ end
46
+
37
47
  next if types.blank?
38
48
 
39
49
  types = Array(types)
@@ -68,10 +68,13 @@ class Eye::Group
68
68
  def send_command(command, *args)
69
69
  info "send_command: #{command}"
70
70
 
71
- if command == :delete
72
- delete *args
73
- else
74
- schedule command, *args, "#{command} by user"
71
+ case command
72
+ when :delete
73
+ delete *args
74
+ when :break_chain
75
+ break_chain *args
76
+ else
77
+ schedule command, *args, "#{command} by user"
75
78
  end
76
79
  end
77
80
 
@@ -104,6 +107,12 @@ class Eye::Group
104
107
  async_schedule :signal, sig
105
108
  end
106
109
 
110
+ def break_chain
111
+ info "break chain"
112
+ scheduler.clear_pending_list
113
+ @chain_breaker = true
114
+ end
115
+
107
116
  def clear
108
117
  @processes = Eye::Utils::AliveArray.new
109
118
  end
@@ -7,6 +7,9 @@ private
7
7
 
8
8
  @chain_processes_count = @processes.size
9
9
  @chain_processes_current = 0
10
+ @chain_breaker = false
11
+
12
+ started_at = Time.now
10
13
 
11
14
  @processes.each do | process |
12
15
  chain_schedule_process(process, type, command, *args)
@@ -15,11 +18,16 @@ private
15
18
 
16
19
  # to skip last sleep
17
20
  break if @chain_processes_current.to_i == @chain_processes_count.to_i
21
+ break if @chain_breaker
18
22
 
19
23
  # wait next process
20
24
  sleep grace.to_f
25
+
26
+ break if @chain_breaker
21
27
  end
22
28
 
29
+ debug "chain finished #{Time.now - started_at}s"
30
+
23
31
  @chain_processes_count = nil
24
32
  @chain_processes_current = nil
25
33
  end
@@ -29,6 +37,9 @@ private
29
37
 
30
38
  if type == :sync
31
39
  # sync command, with waiting
40
+ # this is very hackety, because call method of the process without its scheduler
41
+ # need to provide some scheduler future
42
+ process.last_scheduled_reason = self.last_scheduled_reason
32
43
  process.send(command, *args)
33
44
  else
34
45
  # async command
@@ -1,7 +1,7 @@
1
1
  class Eye::Notify
2
2
  include Celluloid
3
3
  include Eye::Logger::Helpers
4
- extend Eye::Checker::Validation
4
+ extend Eye::Dsl::Validation
5
5
 
6
6
  autoload :Mail, 'eye/notify/mail'
7
7
  autoload :Jabber, 'eye/notify/jabber'
@@ -13,7 +13,7 @@ class Eye::Notify::Mail < Eye::Notify
13
13
  param :domain, String
14
14
  param :user, String
15
15
  param :password, String
16
- param :auth, Symbol # :plain, :login, :cram_md5
16
+ param :auth, Symbol, nil, nil, [:plain, :login, :cram_md5]
17
17
 
18
18
  param :from_mail, String
19
19
  param :from_name, String, nil, 'eye'
@@ -47,8 +47,6 @@ module Eye::Process::Child
47
47
  def remove_childs
48
48
  if childs.present?
49
49
  childs.keys.each{|child_pid| remove_child(child_pid) }
50
- else
51
- debug 'No childs to clear'
52
50
  end
53
51
  end
54
52
 
@@ -17,8 +17,8 @@ module Eye::Process::Commands
17
17
  debug "process (#{self.pid}) ok started"
18
18
  switch :started
19
19
  else
20
- debug "process (#{self.pid}) failed to start (#{result[:error].inspect})"
21
- if self.pid && Eye::System.pid_alive?(self.pid)
20
+ error "process (#{self.pid}) failed to start (#{result[:error].inspect})"
21
+ if process_realy_running?
22
22
  warn "kill, what remains from process (#{self.pid}), because its failed to start (without pid_file impossible to monitoring)"
23
23
  send_signal(:KILL)
24
24
  sleep 0.2 # little grace
@@ -53,7 +53,7 @@ module Eye::Process::Commands
53
53
  switch :stopped
54
54
 
55
55
  if control_pid?
56
- info "delete pid_file: #{self[:pid_file]}"
56
+ info "delete pid_file: #{self[:pid_file_ex]}"
57
57
  clear_pid_file
58
58
  end
59
59
 
@@ -135,7 +135,7 @@ private
135
135
 
136
136
  sleep self[:stop_grace].to_f
137
137
 
138
- # if process not die here, by default we kill it force
138
+ # if process not die here, by default we force kill it
139
139
  if process_realy_running?
140
140
  warn "process not die after TERM and stop_grace #{self[:stop_grace].to_f}s, so send KILL"
141
141
  send_signal(:KILL)
@@ -188,17 +188,14 @@ private
188
188
  sleep self[:start_grace].to_f
189
189
 
190
190
  unless process_realy_running?
191
- error "process with pid (#{self.pid}) not found, may be crashed (#{check_logs_str})"
191
+ error "process with pid(#{self.pid}) not found, may be crashed (#{check_logs_str})"
192
192
  return {:error => :not_realy_running}
193
193
  end
194
194
 
195
- begin
196
- save_pid_to_file
197
- rescue => ex
198
- error "save pid to file raised with #{ex.inspect}"
199
- return {:error => :cant_write_pid}
195
+ unless failsafe_save_pid
196
+ return {:error => :cant_write_pid}
200
197
  end
201
-
198
+
202
199
  res
203
200
  end
204
201
 
@@ -226,17 +223,17 @@ private
226
223
  sleep self[:start_grace].to_f
227
224
 
228
225
  unless set_pid_from_file
229
- error "pid_file(#{self[:pid_file]}) does not appears after start_grace #{self[:start_grace].to_f}, check start_command, or tune start_grace (eye dont know what to monitor without pid)"
226
+ error "pid_file(#{self[:pid_file_ex]}) does not appears after start_grace #{self[:start_grace].to_f}, check start_command, or tune start_grace (eye dont know what to monitor without pid)"
230
227
  return {:error => :pid_not_found}
231
228
  end
232
229
 
233
230
  unless process_realy_running?
234
- error "process in pid_file(#{self[:pid_file]})(#{self.pid}) not found, maybe process do not write there actual pid, or just crashed (#{check_logs_str})"
231
+ error "process in pid_file(#{self[:pid_file_ex]})(#{self.pid}) not found, maybe process do not write there actual pid, or just crashed (#{check_logs_str})"
235
232
  return {:error => :not_realy_running}
236
233
  end
237
234
 
238
235
  res[:pid] = self.pid
239
- info "process get pid:#{res[:pid]}"
236
+ info "process get pid:#{res[:pid]}, pid_file #{self[:pid_file_ex]}"
240
237
  res
241
238
  end
242
239
 
@@ -38,7 +38,7 @@ private
38
38
 
39
39
  # check that process runned
40
40
  unless process_realy_running?
41
- warn 'check_alive: process not found'
41
+ warn "check_alive: process(#{self.pid}) not found!"
42
42
  notify :warn, 'crashed!'
43
43
  switch :crashed
44
44
  else
@@ -49,14 +49,20 @@ private
49
49
  msg = "check_alive: pid_file(#{self[:pid_file]}) changes by itself (pid:#{self.pid}) => (pid:#{ppid})"
50
50
  if control_pid?
51
51
  msg += ", not correct, pid_file is under eye control, so rewrited back pid:#{self.pid}"
52
- save_pid_to_file rescue msg += ', (Can`t rewrite pid_file O_o)'
52
+ unless failsafe_save_pid
53
+ msg += ', (Can`t rewrite pid_file O_o)'
54
+ end
53
55
  else
54
56
  if ppid == nil
55
57
  msg += ', rewrited because empty'
56
- save_pid_to_file rescue msg += ', (Can`t rewrite pid_file O_o)'
58
+ unless failsafe_save_pid
59
+ msg += ', (Can`t rewrite pid_file O_o)'
60
+ end
57
61
  elsif (Time.now - pid_file_ctime > REWRITE_FACKUP_PIDFILE_PERIOD)
58
62
  msg += ", > #{REWRITE_FACKUP_PIDFILE_PERIOD.inspect} ago, so rewrited (even if pid_file not under eye control)"
59
- save_pid_to_file rescue msg += ', (Can`t rewrite pid_file O_o)'
63
+ unless failsafe_save_pid
64
+ msg += ', (Can`t rewrite pid_file O_o)'
65
+ end
60
66
  else
61
67
  msg += ', not under eye control, so ignored'
62
68
  end
@@ -68,18 +74,6 @@ private
68
74
  end
69
75
  end
70
76
 
71
- def failsafe_load_pid
72
- pid = load_pid_from_file
73
-
74
- if !pid
75
- # this is can be symlink changed case
76
- sleep 0.1
77
- pid = load_pid_from_file
78
- end
79
-
80
- pid
81
- end
82
-
83
77
  def check_crash
84
78
  if down?
85
79
  if self[:keep_alive] && !@flapping
@@ -3,18 +3,23 @@ module Eye::Process::Scheduler
3
3
  # ex: schedule :update_config, config, "reason: update_config"
4
4
  def schedule(command, *args, &block)
5
5
  if scheduler.alive?
6
+ unless self.respond_to?(command)
7
+ warn "object not support :#{command} to schedule"
8
+ return
9
+ end
10
+
6
11
  reason = if args.present? && [String, Symbol].include?(args[-1].class)
7
12
  args.pop
8
13
  end
9
14
 
10
- info "schedule :#{command} (#{reason})"
15
+ info "schedule :#{command} #{reason ? "(reason: #{reason})" : nil}"
11
16
  scheduler.add_wo_dups(:scheduled_action, command, {:args => args, :reason => reason}, &block)
12
17
  end
13
18
  end
14
19
 
15
20
  def scheduled_action(command, h = {}, &block)
16
21
  reason = h.delete(:reason)
17
- info "=> #{command} #{h[:args].present? ? "#{h[:args]*',' }" : nil}(#{reason})"
22
+ info "=> #{command} #{h[:args].present? ? "#{h[:args]*',' }" : nil} #{reason ? "(reason: #{reason})" : nil}"
18
23
 
19
24
  @current_scheduled_command = command
20
25
  @last_scheduled_command = command
@@ -47,4 +52,4 @@ private
47
52
  @scheduler ||= Eye::Utils::CelluloidChain.new(current_actor)
48
53
  end
49
54
 
50
- end
55
+ end
@@ -1,4 +1,3 @@
1
- gem 'state_machine'
2
1
  require 'state_machine'
3
2
 
4
3
  class Eye::Process
@@ -82,7 +81,7 @@ class Eye::Process
82
81
 
83
82
  def log_transition(transition)
84
83
  @states_history.push transition.to_name, @state_reason
85
- info "switch :#{transition.event} [:#{transition.from_name} => :#{transition.to_name}] #{@state_reason ? "(#{@state_reason})" : nil}"
84
+ info "switch :#{transition.event} [:#{transition.from_name} => :#{transition.to_name}] #{@state_reason ? "(reason: #{@state_reason})" : nil}"
86
85
  end
87
86
 
88
87
  def upd_for_triggers(transition)
@@ -18,9 +18,10 @@ module Eye::Process::System
18
18
  File.open(self[:pid_file_ex], 'w') do |f|
19
19
  f.write self.pid
20
20
  end
21
+ true
22
+ else
23
+ false
21
24
  end
22
-
23
- true
24
25
  end
25
26
 
26
27
  def clear_pid_file
@@ -35,16 +36,26 @@ module Eye::Process::System
35
36
  end
36
37
 
37
38
  def process_realy_running?
38
- Eye::System.pid_alive?(self.pid) if self.pid
39
+ if self.pid
40
+ res = Eye::System.check_pid_alive(self.pid)
41
+ if res[:error] && res[:error].class != Errno::ESRCH
42
+ error "process_realy_running?: check_pid_alive returns '#{res[:error].message}'"
43
+ end
44
+ res[:result]
45
+ else
46
+ debug "process_realy_running?: called without pid"
47
+ nil
48
+ end
39
49
  end
40
50
 
41
51
  def send_signal(code)
42
- debug "send signal #{code}"
43
-
44
52
  res = Eye::System.send_signal(self.pid, code)
45
- error(res[:message]) if res[:status] != :ok
46
53
 
47
- res[:status] == :ok
54
+ msg = "send_signal #{code} to #{self.pid}"
55
+ msg += ", error<#{res[:error]}>" if res[:error]
56
+ info msg
57
+
58
+ res[:result] == :ok
48
59
  end
49
60
 
50
61
  # non blocking actor timeout
@@ -56,6 +67,26 @@ module Eye::Process::System
56
67
  defer{ Eye::System::execute cmd, cfg }
57
68
  end
58
69
 
70
+ def failsafe_load_pid
71
+ pid = load_pid_from_file
72
+
73
+ if !pid
74
+ # this is can be symlink changed case
75
+ sleep 0.1
76
+ pid = load_pid_from_file
77
+ end
78
+
79
+ pid
80
+ end
81
+
82
+ def failsafe_save_pid
83
+ save_pid_to_file
84
+ true
85
+ rescue => ex
86
+ error "failsafe_save_pid: #{ex.message}"
87
+ false
88
+ end
89
+
59
90
  private
60
91
 
61
92
  def wait_for_condition_sync(timeout, step, &block)