eye 0.2.1 → 0.2.2

Sign up to get free protection for your applications and to get access to all the features.
Files changed (53) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +24 -17
  3. data/bin/eye +5 -0
  4. data/examples/notify.eye +18 -0
  5. data/examples/process_thin.rb +29 -0
  6. data/examples/sidekiq.eye +5 -1
  7. data/examples/test.eye +24 -17
  8. data/examples/thin-farm.eye +29 -0
  9. data/examples/unicorn.eye +2 -0
  10. data/eye.gemspec +1 -1
  11. data/lib/eye.rb +1 -1
  12. data/lib/eye/checker.rb +3 -5
  13. data/lib/eye/checker/http.rb +1 -2
  14. data/lib/eye/checker/socket.rb +2 -2
  15. data/lib/eye/controller/commands.rb +2 -0
  16. data/lib/eye/controller/send_command.rb +8 -2
  17. data/lib/eye/dsl.rb +1 -0
  18. data/lib/eye/dsl/opts.rb +36 -2
  19. data/lib/eye/dsl/pure_opts.rb +17 -39
  20. data/lib/eye/{checker → dsl}/validation.rb +15 -5
  21. data/lib/eye/group.rb +13 -4
  22. data/lib/eye/group/chain.rb +11 -0
  23. data/lib/eye/notify.rb +1 -1
  24. data/lib/eye/notify/mail.rb +1 -1
  25. data/lib/eye/process/child.rb +0 -2
  26. data/lib/eye/process/commands.rb +11 -14
  27. data/lib/eye/process/monitor.rb +10 -16
  28. data/lib/eye/process/scheduler.rb +8 -3
  29. data/lib/eye/process/states.rb +1 -2
  30. data/lib/eye/process/system.rb +38 -7
  31. data/lib/eye/process/watchers.rb +3 -3
  32. data/lib/eye/settings.rb +1 -1
  33. data/lib/eye/system.rb +33 -16
  34. data/lib/eye/trigger.rb +1 -1
  35. data/lib/eye/utils/celluloid_chain.rb +2 -0
  36. data/spec/checker/cpu_spec.rb +1 -1
  37. data/spec/checker/http_spec.rb +2 -2
  38. data/spec/checker/memory_spec.rb +2 -2
  39. data/spec/checker_spec.rb +1 -1
  40. data/spec/controller/controller_spec.rb +6 -0
  41. data/spec/controller/find_objects_spec.rb +6 -0
  42. data/spec/controller/intergration_spec.rb +24 -0
  43. data/spec/dsl/checks_spec.rb +2 -2
  44. data/spec/dsl/notify_spec.rb +12 -3
  45. data/spec/dsl/with_server_spec.rb +26 -2
  46. data/spec/process/checks/child_checks_spec.rb +1 -1
  47. data/spec/process/checks/memory_spec.rb +14 -0
  48. data/spec/process/scheduler_spec.rb +8 -0
  49. data/spec/process/system_spec.rb +27 -4
  50. data/spec/spec_helper.rb +3 -1
  51. data/spec/system_spec.rb +12 -5
  52. data/spec/utils/celluloid_chain_spec.rb +8 -0
  53. metadata +8 -5
@@ -66,49 +66,10 @@ class Eye::Dsl::PureOpts
66
66
  []
67
67
  end
68
68
 
69
- # execute part of config on particular server
70
- # array of strings
71
- # regexp
72
- # string
73
- def with_server(glob = nil, &block)
74
- on_server = true
75
-
76
- if glob.present?
77
- host = Eye::System.host
78
-
79
- if glob.is_a?(Array)
80
- on_server = !!glob.any?{|elem| elem == host}
81
- elsif glob.is_a?(Regexp)
82
- on_server = !!host.match(glob)
83
- elsif glob.is_a?(String) || glob.is_a?(Symbol)
84
- on_server = (host == glob.to_s)
85
- end
86
- end
87
-
88
- with_condition(on_server, &block)
89
-
90
- on_server
91
- end
92
-
93
69
  def with_condition(cond = true, &block)
94
70
  self.instance_eval(&block) if cond && block
95
71
  end
96
72
 
97
- def self.with_parsed_file(file_name)
98
- saved_parsed_filename = Eye.parsed_filename
99
-
100
- require 'pathname'
101
-
102
- real_filename = Eye.parsed_filename && File.symlink?(Eye.parsed_filename) ? File.readlink(Eye.parsed_filename) : Eye.parsed_filename
103
- dirname = File.dirname(real_filename) rescue nil
104
- path = Pathname.new(file_name).expand_path(dirname).to_s
105
-
106
- Eye.parsed_filename = path
107
- yield path
108
- ensure
109
- Eye.parsed_filename = saved_parsed_filename
110
- end
111
-
112
73
  def include(proc, *args)
113
74
  if proc.is_a?(String)
114
75
  self.class.with_parsed_file(proc) do |path|
@@ -129,4 +90,21 @@ class Eye::Dsl::PureOpts
129
90
  end
130
91
  end
131
92
 
93
+ private
94
+
95
+ def self.with_parsed_file(file_name)
96
+ saved_parsed_filename = Eye.parsed_filename
97
+
98
+ require 'pathname'
99
+
100
+ real_filename = Eye.parsed_filename && File.symlink?(Eye.parsed_filename) ? File.readlink(Eye.parsed_filename) : Eye.parsed_filename
101
+ dirname = File.dirname(real_filename) rescue nil
102
+ path = Pathname.new(file_name).expand_path(dirname).to_s
103
+
104
+ Eye.parsed_filename = path
105
+ yield path
106
+ ensure
107
+ Eye.parsed_filename = saved_parsed_filename
108
+ end
109
+
132
110
  end
@@ -1,24 +1,27 @@
1
- module Eye::Checker::Validation
1
+ module Eye::Dsl::Validation
2
2
  class Error < Exception; end
3
3
 
4
4
  def inherited(subclass)
5
5
  subclass.validates = self.validates.clone
6
6
  subclass.should_bes = self.should_bes.clone
7
7
  subclass.defaults = self.defaults.clone
8
+ subclass.variants = self.variants.clone
8
9
  end
9
10
 
10
- attr_accessor :validates, :should_bes, :defaults
11
+ attr_accessor :validates, :should_bes, :defaults, :variants
11
12
 
12
13
  def validates; @validates ||= {}; end
13
14
  def should_bes; @should_bes ||= []; end
14
15
  def defaults; @defaults ||= {}; end
16
+ def variants; @variants ||= {}; end
15
17
 
16
- def param(param, types = [], should_be = false, default = nil)
18
+ def param(param, types = [], should_be = false, default = nil, _variants = nil)
17
19
  param = param.to_sym
18
20
 
19
21
  validates[param] = types
20
22
  should_bes << param if should_be
21
23
  defaults[param] = default
24
+ variants[param] = _variants
22
25
 
23
26
  define_method "#{param}" do
24
27
  @options[param.to_sym] || default
@@ -27,13 +30,20 @@ module Eye::Checker::Validation
27
30
 
28
31
  def validate(options = {})
29
32
  options.each do |param, value|
30
- types = validates[param.to_sym]
33
+ param = param.to_sym
34
+ types = validates[param]
31
35
  unless types
32
- if param.to_sym != :type
36
+ if param != :type
33
37
  raise Error, "#{self.name} unknown param :#{param} value #{value.inspect}"
34
38
  end
35
39
  end
36
40
 
41
+ if self.variants[param]
42
+ if value && !value.is_a?(Proc) && !self.variants[param].include?(value)
43
+ raise Error, "#{value.inspect} should within #{self.variants[param].inspect}"
44
+ end
45
+ end
46
+
37
47
  next if types.blank?
38
48
 
39
49
  types = Array(types)
@@ -68,10 +68,13 @@ class Eye::Group
68
68
  def send_command(command, *args)
69
69
  info "send_command: #{command}"
70
70
 
71
- if command == :delete
72
- delete *args
73
- else
74
- schedule command, *args, "#{command} by user"
71
+ case command
72
+ when :delete
73
+ delete *args
74
+ when :break_chain
75
+ break_chain *args
76
+ else
77
+ schedule command, *args, "#{command} by user"
75
78
  end
76
79
  end
77
80
 
@@ -104,6 +107,12 @@ class Eye::Group
104
107
  async_schedule :signal, sig
105
108
  end
106
109
 
110
+ def break_chain
111
+ info "break chain"
112
+ scheduler.clear_pending_list
113
+ @chain_breaker = true
114
+ end
115
+
107
116
  def clear
108
117
  @processes = Eye::Utils::AliveArray.new
109
118
  end
@@ -7,6 +7,9 @@ private
7
7
 
8
8
  @chain_processes_count = @processes.size
9
9
  @chain_processes_current = 0
10
+ @chain_breaker = false
11
+
12
+ started_at = Time.now
10
13
 
11
14
  @processes.each do | process |
12
15
  chain_schedule_process(process, type, command, *args)
@@ -15,11 +18,16 @@ private
15
18
 
16
19
  # to skip last sleep
17
20
  break if @chain_processes_current.to_i == @chain_processes_count.to_i
21
+ break if @chain_breaker
18
22
 
19
23
  # wait next process
20
24
  sleep grace.to_f
25
+
26
+ break if @chain_breaker
21
27
  end
22
28
 
29
+ debug "chain finished #{Time.now - started_at}s"
30
+
23
31
  @chain_processes_count = nil
24
32
  @chain_processes_current = nil
25
33
  end
@@ -29,6 +37,9 @@ private
29
37
 
30
38
  if type == :sync
31
39
  # sync command, with waiting
40
+ # this is very hackety, because call method of the process without its scheduler
41
+ # need to provide some scheduler future
42
+ process.last_scheduled_reason = self.last_scheduled_reason
32
43
  process.send(command, *args)
33
44
  else
34
45
  # async command
@@ -1,7 +1,7 @@
1
1
  class Eye::Notify
2
2
  include Celluloid
3
3
  include Eye::Logger::Helpers
4
- extend Eye::Checker::Validation
4
+ extend Eye::Dsl::Validation
5
5
 
6
6
  autoload :Mail, 'eye/notify/mail'
7
7
  autoload :Jabber, 'eye/notify/jabber'
@@ -13,7 +13,7 @@ class Eye::Notify::Mail < Eye::Notify
13
13
  param :domain, String
14
14
  param :user, String
15
15
  param :password, String
16
- param :auth, Symbol # :plain, :login, :cram_md5
16
+ param :auth, Symbol, nil, nil, [:plain, :login, :cram_md5]
17
17
 
18
18
  param :from_mail, String
19
19
  param :from_name, String, nil, 'eye'
@@ -47,8 +47,6 @@ module Eye::Process::Child
47
47
  def remove_childs
48
48
  if childs.present?
49
49
  childs.keys.each{|child_pid| remove_child(child_pid) }
50
- else
51
- debug 'No childs to clear'
52
50
  end
53
51
  end
54
52
 
@@ -17,8 +17,8 @@ module Eye::Process::Commands
17
17
  debug "process (#{self.pid}) ok started"
18
18
  switch :started
19
19
  else
20
- debug "process (#{self.pid}) failed to start (#{result[:error].inspect})"
21
- if self.pid && Eye::System.pid_alive?(self.pid)
20
+ error "process (#{self.pid}) failed to start (#{result[:error].inspect})"
21
+ if process_realy_running?
22
22
  warn "kill, what remains from process (#{self.pid}), because its failed to start (without pid_file impossible to monitoring)"
23
23
  send_signal(:KILL)
24
24
  sleep 0.2 # little grace
@@ -53,7 +53,7 @@ module Eye::Process::Commands
53
53
  switch :stopped
54
54
 
55
55
  if control_pid?
56
- info "delete pid_file: #{self[:pid_file]}"
56
+ info "delete pid_file: #{self[:pid_file_ex]}"
57
57
  clear_pid_file
58
58
  end
59
59
 
@@ -135,7 +135,7 @@ private
135
135
 
136
136
  sleep self[:stop_grace].to_f
137
137
 
138
- # if process not die here, by default we kill it force
138
+ # if process not die here, by default we force kill it
139
139
  if process_realy_running?
140
140
  warn "process not die after TERM and stop_grace #{self[:stop_grace].to_f}s, so send KILL"
141
141
  send_signal(:KILL)
@@ -188,17 +188,14 @@ private
188
188
  sleep self[:start_grace].to_f
189
189
 
190
190
  unless process_realy_running?
191
- error "process with pid (#{self.pid}) not found, may be crashed (#{check_logs_str})"
191
+ error "process with pid(#{self.pid}) not found, may be crashed (#{check_logs_str})"
192
192
  return {:error => :not_realy_running}
193
193
  end
194
194
 
195
- begin
196
- save_pid_to_file
197
- rescue => ex
198
- error "save pid to file raised with #{ex.inspect}"
199
- return {:error => :cant_write_pid}
195
+ unless failsafe_save_pid
196
+ return {:error => :cant_write_pid}
200
197
  end
201
-
198
+
202
199
  res
203
200
  end
204
201
 
@@ -226,17 +223,17 @@ private
226
223
  sleep self[:start_grace].to_f
227
224
 
228
225
  unless set_pid_from_file
229
- error "pid_file(#{self[:pid_file]}) does not appears after start_grace #{self[:start_grace].to_f}, check start_command, or tune start_grace (eye dont know what to monitor without pid)"
226
+ error "pid_file(#{self[:pid_file_ex]}) does not appears after start_grace #{self[:start_grace].to_f}, check start_command, or tune start_grace (eye dont know what to monitor without pid)"
230
227
  return {:error => :pid_not_found}
231
228
  end
232
229
 
233
230
  unless process_realy_running?
234
- error "process in pid_file(#{self[:pid_file]})(#{self.pid}) not found, maybe process do not write there actual pid, or just crashed (#{check_logs_str})"
231
+ error "process in pid_file(#{self[:pid_file_ex]})(#{self.pid}) not found, maybe process do not write there actual pid, or just crashed (#{check_logs_str})"
235
232
  return {:error => :not_realy_running}
236
233
  end
237
234
 
238
235
  res[:pid] = self.pid
239
- info "process get pid:#{res[:pid]}"
236
+ info "process get pid:#{res[:pid]}, pid_file #{self[:pid_file_ex]}"
240
237
  res
241
238
  end
242
239
 
@@ -38,7 +38,7 @@ private
38
38
 
39
39
  # check that process runned
40
40
  unless process_realy_running?
41
- warn 'check_alive: process not found'
41
+ warn "check_alive: process(#{self.pid}) not found!"
42
42
  notify :warn, 'crashed!'
43
43
  switch :crashed
44
44
  else
@@ -49,14 +49,20 @@ private
49
49
  msg = "check_alive: pid_file(#{self[:pid_file]}) changes by itself (pid:#{self.pid}) => (pid:#{ppid})"
50
50
  if control_pid?
51
51
  msg += ", not correct, pid_file is under eye control, so rewrited back pid:#{self.pid}"
52
- save_pid_to_file rescue msg += ', (Can`t rewrite pid_file O_o)'
52
+ unless failsafe_save_pid
53
+ msg += ', (Can`t rewrite pid_file O_o)'
54
+ end
53
55
  else
54
56
  if ppid == nil
55
57
  msg += ', rewrited because empty'
56
- save_pid_to_file rescue msg += ', (Can`t rewrite pid_file O_o)'
58
+ unless failsafe_save_pid
59
+ msg += ', (Can`t rewrite pid_file O_o)'
60
+ end
57
61
  elsif (Time.now - pid_file_ctime > REWRITE_FACKUP_PIDFILE_PERIOD)
58
62
  msg += ", > #{REWRITE_FACKUP_PIDFILE_PERIOD.inspect} ago, so rewrited (even if pid_file not under eye control)"
59
- save_pid_to_file rescue msg += ', (Can`t rewrite pid_file O_o)'
63
+ unless failsafe_save_pid
64
+ msg += ', (Can`t rewrite pid_file O_o)'
65
+ end
60
66
  else
61
67
  msg += ', not under eye control, so ignored'
62
68
  end
@@ -68,18 +74,6 @@ private
68
74
  end
69
75
  end
70
76
 
71
- def failsafe_load_pid
72
- pid = load_pid_from_file
73
-
74
- if !pid
75
- # this is can be symlink changed case
76
- sleep 0.1
77
- pid = load_pid_from_file
78
- end
79
-
80
- pid
81
- end
82
-
83
77
  def check_crash
84
78
  if down?
85
79
  if self[:keep_alive] && !@flapping
@@ -3,18 +3,23 @@ module Eye::Process::Scheduler
3
3
  # ex: schedule :update_config, config, "reason: update_config"
4
4
  def schedule(command, *args, &block)
5
5
  if scheduler.alive?
6
+ unless self.respond_to?(command)
7
+ warn "object not support :#{command} to schedule"
8
+ return
9
+ end
10
+
6
11
  reason = if args.present? && [String, Symbol].include?(args[-1].class)
7
12
  args.pop
8
13
  end
9
14
 
10
- info "schedule :#{command} (#{reason})"
15
+ info "schedule :#{command} #{reason ? "(reason: #{reason})" : nil}"
11
16
  scheduler.add_wo_dups(:scheduled_action, command, {:args => args, :reason => reason}, &block)
12
17
  end
13
18
  end
14
19
 
15
20
  def scheduled_action(command, h = {}, &block)
16
21
  reason = h.delete(:reason)
17
- info "=> #{command} #{h[:args].present? ? "#{h[:args]*',' }" : nil}(#{reason})"
22
+ info "=> #{command} #{h[:args].present? ? "#{h[:args]*',' }" : nil} #{reason ? "(reason: #{reason})" : nil}"
18
23
 
19
24
  @current_scheduled_command = command
20
25
  @last_scheduled_command = command
@@ -47,4 +52,4 @@ private
47
52
  @scheduler ||= Eye::Utils::CelluloidChain.new(current_actor)
48
53
  end
49
54
 
50
- end
55
+ end
@@ -1,4 +1,3 @@
1
- gem 'state_machine'
2
1
  require 'state_machine'
3
2
 
4
3
  class Eye::Process
@@ -82,7 +81,7 @@ class Eye::Process
82
81
 
83
82
  def log_transition(transition)
84
83
  @states_history.push transition.to_name, @state_reason
85
- info "switch :#{transition.event} [:#{transition.from_name} => :#{transition.to_name}] #{@state_reason ? "(#{@state_reason})" : nil}"
84
+ info "switch :#{transition.event} [:#{transition.from_name} => :#{transition.to_name}] #{@state_reason ? "(reason: #{@state_reason})" : nil}"
86
85
  end
87
86
 
88
87
  def upd_for_triggers(transition)
@@ -18,9 +18,10 @@ module Eye::Process::System
18
18
  File.open(self[:pid_file_ex], 'w') do |f|
19
19
  f.write self.pid
20
20
  end
21
+ true
22
+ else
23
+ false
21
24
  end
22
-
23
- true
24
25
  end
25
26
 
26
27
  def clear_pid_file
@@ -35,16 +36,26 @@ module Eye::Process::System
35
36
  end
36
37
 
37
38
  def process_realy_running?
38
- Eye::System.pid_alive?(self.pid) if self.pid
39
+ if self.pid
40
+ res = Eye::System.check_pid_alive(self.pid)
41
+ if res[:error] && res[:error].class != Errno::ESRCH
42
+ error "process_realy_running?: check_pid_alive returns '#{res[:error].message}'"
43
+ end
44
+ res[:result]
45
+ else
46
+ debug "process_realy_running?: called without pid"
47
+ nil
48
+ end
39
49
  end
40
50
 
41
51
  def send_signal(code)
42
- debug "send signal #{code}"
43
-
44
52
  res = Eye::System.send_signal(self.pid, code)
45
- error(res[:message]) if res[:status] != :ok
46
53
 
47
- res[:status] == :ok
54
+ msg = "send_signal #{code} to #{self.pid}"
55
+ msg += ", error<#{res[:error]}>" if res[:error]
56
+ info msg
57
+
58
+ res[:result] == :ok
48
59
  end
49
60
 
50
61
  # non blocking actor timeout
@@ -56,6 +67,26 @@ module Eye::Process::System
56
67
  defer{ Eye::System::execute cmd, cfg }
57
68
  end
58
69
 
70
+ def failsafe_load_pid
71
+ pid = load_pid_from_file
72
+
73
+ if !pid
74
+ # this is can be symlink changed case
75
+ sleep 0.1
76
+ pid = load_pid_from_file
77
+ end
78
+
79
+ pid
80
+ end
81
+
82
+ def failsafe_save_pid
83
+ save_pid_to_file
84
+ true
85
+ rescue => ex
86
+ error "failsafe_save_pid: #{ex.message}"
87
+ false
88
+ end
89
+
59
90
  private
60
91
 
61
92
  def wait_for_condition_sync(timeout, step, &block)