ace-eye 0.6.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (114) hide show
  1. checksums.yaml +7 -0
  2. data/.gitignore +38 -0
  3. data/.rspec +2 -0
  4. data/.travis.yml +6 -0
  5. data/CHANGES.md +77 -0
  6. data/Gemfile +6 -0
  7. data/LICENSE +22 -0
  8. data/README.md +212 -0
  9. data/Rakefile +35 -0
  10. data/bin/eye +5 -0
  11. data/bin/loader_eye +72 -0
  12. data/bin/runner +16 -0
  13. data/examples/dependency.eye +17 -0
  14. data/examples/notify.eye +19 -0
  15. data/examples/plugin/README.md +15 -0
  16. data/examples/plugin/main.eye +15 -0
  17. data/examples/plugin/plugin.rb +63 -0
  18. data/examples/process_thin.rb +29 -0
  19. data/examples/processes/em.rb +57 -0
  20. data/examples/processes/forking.rb +20 -0
  21. data/examples/processes/sample.rb +144 -0
  22. data/examples/processes/thin.ru +12 -0
  23. data/examples/puma.eye +29 -0
  24. data/examples/rbenv.eye +11 -0
  25. data/examples/sidekiq.eye +23 -0
  26. data/examples/test.eye +87 -0
  27. data/examples/thin-farm.eye +30 -0
  28. data/examples/unicorn.eye +39 -0
  29. data/eye.gemspec +40 -0
  30. data/lib/eye.rb +28 -0
  31. data/lib/eye/application.rb +73 -0
  32. data/lib/eye/checker.rb +258 -0
  33. data/lib/eye/checker/children_count.rb +44 -0
  34. data/lib/eye/checker/children_memory.rb +12 -0
  35. data/lib/eye/checker/cpu.rb +17 -0
  36. data/lib/eye/checker/cputime.rb +13 -0
  37. data/lib/eye/checker/file_ctime.rb +24 -0
  38. data/lib/eye/checker/file_size.rb +34 -0
  39. data/lib/eye/checker/file_touched.rb +15 -0
  40. data/lib/eye/checker/http.rb +96 -0
  41. data/lib/eye/checker/memory.rb +17 -0
  42. data/lib/eye/checker/nop.rb +6 -0
  43. data/lib/eye/checker/runtime.rb +18 -0
  44. data/lib/eye/checker/socket.rb +159 -0
  45. data/lib/eye/child_process.rb +101 -0
  46. data/lib/eye/cli.rb +185 -0
  47. data/lib/eye/cli/commands.rb +78 -0
  48. data/lib/eye/cli/render.rb +130 -0
  49. data/lib/eye/cli/server.rb +93 -0
  50. data/lib/eye/client.rb +32 -0
  51. data/lib/eye/config.rb +91 -0
  52. data/lib/eye/control.rb +2 -0
  53. data/lib/eye/controller.rb +54 -0
  54. data/lib/eye/controller/commands.rb +88 -0
  55. data/lib/eye/controller/helpers.rb +101 -0
  56. data/lib/eye/controller/load.rb +224 -0
  57. data/lib/eye/controller/options.rb +18 -0
  58. data/lib/eye/controller/send_command.rb +177 -0
  59. data/lib/eye/controller/status.rb +72 -0
  60. data/lib/eye/dsl.rb +53 -0
  61. data/lib/eye/dsl/application_opts.rb +39 -0
  62. data/lib/eye/dsl/chain.rb +12 -0
  63. data/lib/eye/dsl/child_process_opts.rb +13 -0
  64. data/lib/eye/dsl/config_opts.rb +55 -0
  65. data/lib/eye/dsl/group_opts.rb +32 -0
  66. data/lib/eye/dsl/helpers.rb +20 -0
  67. data/lib/eye/dsl/main.rb +51 -0
  68. data/lib/eye/dsl/opts.rb +151 -0
  69. data/lib/eye/dsl/process_opts.rb +36 -0
  70. data/lib/eye/dsl/pure_opts.rb +121 -0
  71. data/lib/eye/dsl/validation.rb +88 -0
  72. data/lib/eye/group.rb +140 -0
  73. data/lib/eye/group/chain.rb +81 -0
  74. data/lib/eye/loader.rb +10 -0
  75. data/lib/eye/local.rb +100 -0
  76. data/lib/eye/logger.rb +104 -0
  77. data/lib/eye/notify.rb +118 -0
  78. data/lib/eye/notify/jabber.rb +30 -0
  79. data/lib/eye/notify/mail.rb +48 -0
  80. data/lib/eye/process.rb +85 -0
  81. data/lib/eye/process/children.rb +60 -0
  82. data/lib/eye/process/commands.rb +280 -0
  83. data/lib/eye/process/config.rb +81 -0
  84. data/lib/eye/process/controller.rb +73 -0
  85. data/lib/eye/process/data.rb +78 -0
  86. data/lib/eye/process/monitor.rb +108 -0
  87. data/lib/eye/process/notify.rb +32 -0
  88. data/lib/eye/process/scheduler.rb +82 -0
  89. data/lib/eye/process/states.rb +86 -0
  90. data/lib/eye/process/states_history.rb +66 -0
  91. data/lib/eye/process/system.rb +97 -0
  92. data/lib/eye/process/trigger.rb +34 -0
  93. data/lib/eye/process/validate.rb +33 -0
  94. data/lib/eye/process/watchers.rb +66 -0
  95. data/lib/eye/reason.rb +20 -0
  96. data/lib/eye/server.rb +60 -0
  97. data/lib/eye/sigar.rb +5 -0
  98. data/lib/eye/system.rb +139 -0
  99. data/lib/eye/system_resources.rb +99 -0
  100. data/lib/eye/trigger.rb +136 -0
  101. data/lib/eye/trigger/check_dependency.rb +30 -0
  102. data/lib/eye/trigger/flapping.rb +41 -0
  103. data/lib/eye/trigger/stop_children.rb +17 -0
  104. data/lib/eye/trigger/transition.rb +15 -0
  105. data/lib/eye/trigger/wait_dependency.rb +49 -0
  106. data/lib/eye/utils.rb +45 -0
  107. data/lib/eye/utils/alive_array.rb +57 -0
  108. data/lib/eye/utils/celluloid_chain.rb +71 -0
  109. data/lib/eye/utils/celluloid_klass.rb +5 -0
  110. data/lib/eye/utils/leak_19.rb +10 -0
  111. data/lib/eye/utils/mini_active_support.rb +111 -0
  112. data/lib/eye/utils/pmap.rb +7 -0
  113. data/lib/eye/utils/tail.rb +20 -0
  114. metadata +398 -0
@@ -0,0 +1,280 @@
1
+ module Eye::Process::Commands
2
+
3
+ def start_process
4
+ debug 'start_process command'
5
+
6
+ switch :starting
7
+
8
+ unless self[:start_command]
9
+ warn 'no :start_command found, unmonitoring'
10
+ switch :unmonitoring, Eye::Reason.new(:no_start_command)
11
+ return :no_start_command
12
+ end
13
+
14
+ result = self[:daemonize] ? daemonize_process : execute_process
15
+
16
+ if !result[:error]
17
+ debug "process <#{self.pid}> started successfully"
18
+ switch :started
19
+ else
20
+ error "process <#{self.pid}> failed to start (#{result[:error].inspect})"
21
+
22
+ if process_really_running?
23
+ warn "killing <#{self.pid}> due to error"
24
+ send_signal(:KILL)
25
+ sleep 0.2 # little grace
26
+ end
27
+
28
+ self.pid = nil
29
+ switch :crashed
30
+ end
31
+
32
+ result
33
+
34
+ rescue StateMachine::InvalidTransition, Eye::Process::StateError => e
35
+ warn "wrong switch '#{e.message}'"
36
+
37
+ :state_error
38
+ end
39
+
40
+ def stop_process
41
+ debug 'stop_process command'
42
+
43
+ switch :stopping
44
+
45
+ kill_process
46
+
47
+ if process_really_running?
48
+ warn "process <#{self.pid}> was not stopped; try checking your command/signals or tuning the stop_timeout/stop_grace values"
49
+
50
+ switch :unmonitoring, Eye::Reason.new(:'not stopped (soft command)')
51
+ nil
52
+
53
+ else
54
+ switch :stopped
55
+
56
+ clear_pid_file if self[:clear_pid] # by default for all
57
+
58
+ true
59
+ end
60
+
61
+ rescue StateMachine::InvalidTransition, Eye::Process::StateError => e
62
+ warn "wrong switch '#{e.message}'"
63
+ nil
64
+ end
65
+
66
+ def restart_process
67
+ debug 'restart_process command'
68
+
69
+ switch :restarting
70
+
71
+ if self[:restart_command]
72
+ execute_restart_command
73
+ sleep_grace(:restart_grace)
74
+ result = check_alive_with_refresh_pid_if_needed
75
+ switch(result ? :restarted : :crashed)
76
+ else
77
+ stop_process
78
+ start_process
79
+ end
80
+
81
+ true
82
+
83
+ rescue StateMachine::InvalidTransition, Eye::Process::StateError => e
84
+ warn "wrong switch '#{e.message}'"
85
+ nil
86
+ end
87
+
88
+ private
89
+
90
+ def kill_process
91
+ unless self.pid
92
+ error 'cannot stop a process without a pid'
93
+ return
94
+ end
95
+
96
+ if self[:stop_command]
97
+ cmd = prepare_command(self[:stop_command])
98
+ info "executing: `#{cmd}` with stop_timeout: #{self[:stop_timeout].to_f}s and stop_grace: #{self[:stop_grace].to_f}s"
99
+ res = execute(cmd, config.merge(:timeout => self[:stop_timeout]))
100
+
101
+ if res[:error]
102
+
103
+ if res[:error].class == Timeout::Error
104
+ error "stop_command failed with #{res[:error].inspect}; try tuning the stop_timeout value"
105
+ else
106
+ error "stop_command failed with #{res[:error].inspect}"
107
+ end
108
+ end
109
+
110
+ sleep_grace(:stop_grace)
111
+
112
+ elsif self[:stop_signals]
113
+ info "executing stop_signals #{self[:stop_signals].inspect}"
114
+ stop_signals = self[:stop_signals].clone
115
+
116
+ signal = stop_signals.shift
117
+ send_signal(signal)
118
+
119
+ while stop_signals.present?
120
+ delay = stop_signals.shift
121
+ signal = stop_signals.shift
122
+
123
+ if wait_for_condition(delay.to_f, 0.3){ !process_really_running? }
124
+ info 'has terminated'
125
+ break
126
+ end
127
+
128
+ send_signal(signal) if signal
129
+ end
130
+
131
+ sleep_grace(:stop_grace)
132
+
133
+ else # default command
134
+ debug "executing: `kill -TERM #{self.pid}` with stop_grace: #{self[:stop_grace].to_f}s"
135
+ send_signal(:TERM)
136
+
137
+ sleep_grace(:stop_grace)
138
+
139
+ # if process not die here, by default we force kill it
140
+ if process_really_running?
141
+ warn "process <#{self.pid}> did not die after TERM, sending KILL"
142
+ send_signal(:KILL)
143
+ sleep 0.1 # little grace
144
+ end
145
+ end
146
+ end
147
+
148
+ def execute_restart_command
149
+ unless self.pid
150
+ error 'cannot restart a process without a pid'
151
+ return
152
+ end
153
+
154
+ cmd = prepare_command(self[:restart_command])
155
+ info "executing: `#{cmd}` with restart_timeout: #{self[:restart_timeout].to_f}s and restart_grace: #{self[:restart_grace].to_f}s"
156
+
157
+ res = execute(cmd, config.merge(:timeout => self[:restart_timeout]))
158
+
159
+ if res[:error]
160
+
161
+ if res[:error].class == Timeout::Error
162
+ error "restart_command failed with #{res[:error].inspect}; try tuning the restart_timeout value"
163
+ else
164
+ error "restart_command failed with #{res[:error].inspect}"
165
+ end
166
+ end
167
+
168
+ res
169
+ end
170
+
171
+ def daemonize_process
172
+ time_before = Time.now
173
+ res = Eye::System.daemonize(self[:start_command], config)
174
+ start_time = Time.now - time_before
175
+
176
+ info "daemonizing: `#{self[:start_command]}` with start_grace: #{self[:start_grace].to_f}s, env: '#{environment_string}', <#{res[:pid]}> (in #{self[:working_dir]})"
177
+
178
+ if res[:error]
179
+
180
+ if res[:error].message == 'Permission denied - open'
181
+ error "daemonize failed with #{res[:error].inspect}; make sure #{[self[:stdout], self[:stderr]]} are writable"
182
+ else
183
+ error "daemonize failed with #{res[:error].inspect}"
184
+ end
185
+
186
+ return {:error => res[:error].inspect}
187
+ end
188
+
189
+ self.pid = res[:pid]
190
+
191
+ unless self.pid
192
+ error 'no pid was returned'
193
+ return {:error => :empty_pid}
194
+ end
195
+
196
+ sleep_grace(:start_grace)
197
+
198
+ unless process_really_running?
199
+ error "process <#{self.pid}> not found, it may have crashed (#{check_logs_str})"
200
+ return {:error => :not_really_running}
201
+ end
202
+
203
+ # if we using leaf child stratedy, pid should be used as last child process
204
+ if self[:use_leaf_child]
205
+ if lpid = Eye::SystemResources.leaf_child(self.pid)
206
+ info "leaf child for <#{self.pid}> found: <#{lpid}>, accepting it!"
207
+ self.parent_pid = self.pid
208
+ self.pid = lpid
209
+ else
210
+ warn "leaf child not found for <#{self.pid}>, skipping it"
211
+ end
212
+ end
213
+
214
+ if control_pid? && !failsafe_save_pid
215
+ return {:error => :cant_write_pid}
216
+ end
217
+
218
+ res
219
+ end
220
+
221
+ def execute_process
222
+ info "executing: `#{self[:start_command]}` with start_timeout: #{config[:start_timeout].to_f}s, start_grace: #{self[:start_grace].to_f}s, env: '#{environment_string}' (in #{self[:working_dir]})"
223
+ time_before = Time.now
224
+
225
+ res = execute(self[:start_command], config.merge(:timeout => config[:start_timeout]))
226
+ start_time = Time.now - time_before
227
+
228
+ if res[:error]
229
+
230
+ if res[:error].message == 'Permission denied - open'
231
+ error "execution failed with #{res[:error].inspect}; ensure that #{[self[:stdout], self[:stderr]]} are writable"
232
+ elsif res[:error].class == Timeout::Error
233
+ error "execution failed with #{res[:error].inspect}; try increasing the start_timeout value (the current value of #{self[:start_timeout]}s seems too short)"
234
+ else
235
+ error "execution failed with #{res[:error].inspect}"
236
+ end
237
+
238
+ return {:error => res[:error].inspect}
239
+ end
240
+
241
+ sleep_grace(:start_grace)
242
+
243
+ unless set_pid_from_file
244
+ error "exit status #{res[:exitstatus]}, pid_file (#{self[:pid_file_ex]}) did not appear within the start_grace period (#{self[:start_grace].to_f}s); check your start_command, or tune the start_grace value (eye expect process to create pid_file in self-daemonization mode)"
245
+ return {:error => :pid_not_found}
246
+ end
247
+
248
+ unless process_really_running?
249
+ error "exit status #{res[:exitstatus]}, process <#{self.pid}> (from #{self[:pid_file_ex]}) was not found; ensure that the pid_file is being updated correctly (#{check_logs_str})"
250
+ return {:error => :not_really_running}
251
+ end
252
+
253
+ res[:pid] = self.pid
254
+ info "exit status #{res[:exitstatus]}, process <#{res[:pid]}> (from #{self[:pid_file_ex]}) was found"
255
+ res
256
+ end
257
+
258
+ def check_logs_str
259
+ if !self[:stdout] && !self[:stderr]
260
+ 'you may want to configure stdout/err/all logs for this process'
261
+ else
262
+ "you should check the process logs #{[self[:stdout], self[:stderr]]}"
263
+ end
264
+ end
265
+
266
+ def prepare_command(command)
267
+ if self.pid
268
+ command.to_s.gsub('{{PID}}', self.pid.to_s).gsub('{PID}', self.pid.to_s)
269
+ else
270
+ command
271
+ end
272
+ end
273
+
274
+ def sleep_grace(grace_name)
275
+ grace = self[grace_name].to_f
276
+ info "sleeping for :#{grace_name} #{grace}"
277
+ sleep grace
278
+ end
279
+
280
+ end
@@ -0,0 +1,81 @@
1
+ module Eye::Process::Config
2
+
3
+ DEFAULTS = {
4
+ :keep_alive => true, # restart when crashed
5
+ :check_alive_period => 5.seconds,
6
+
7
+ :start_timeout => 15.seconds,
8
+ :stop_timeout => 10.seconds,
9
+ :restart_timeout => 10.seconds,
10
+
11
+ :start_grace => 2.5.seconds,
12
+ :stop_grace => 0.5.seconds,
13
+ :restart_grace => 1.second,
14
+
15
+ :daemonize => false,
16
+ :auto_start => true, # auto start on monitor action
17
+
18
+ :children_update_period => 30.seconds,
19
+ :clear_pid => true, # by default clear pid on stop
20
+
21
+ :auto_update_pidfile_grace => 30.seconds,
22
+ :revert_fuckup_pidfile_grace => 120.seconds,
23
+ }
24
+
25
+ def prepare_config(new_config)
26
+ h = DEFAULTS.merge(new_config)
27
+ h[:pid_file_ex] = Eye::System.normalized_file(h[:pid_file], h[:working_dir]) if h[:pid_file]
28
+ h[:checks] = {} if h[:checks].blank?
29
+ h[:triggers] = {} if h[:triggers].blank?
30
+ h[:children_update_period] = h[:monitor_children][:children_update_period] if h[:monitor_children] && h[:monitor_children][:children_update_period]
31
+
32
+ # check speedy flapping by default
33
+ if h[:triggers].blank? || !h[:triggers][:flapping]
34
+ h[:triggers] ||= {}
35
+ h[:triggers][:flapping] = {:type => :flapping, :times => 10, :within => 10.seconds}
36
+ end
37
+
38
+ h[:stdout] = Eye::System.normalized_file(h[:stdout], h[:working_dir]) if h[:stdout]
39
+ h[:stderr] = Eye::System.normalized_file(h[:stderr], h[:working_dir]) if h[:stderr]
40
+ h[:stdall] = Eye::System.normalized_file(h[:stdall], h[:working_dir]) if h[:stdall]
41
+
42
+ h[:environment] = Eye::System.prepare_env(h)
43
+
44
+ h
45
+ end
46
+
47
+ def c(name)
48
+ @config[name]
49
+ end
50
+
51
+ def [](name)
52
+ @config[name]
53
+ end
54
+
55
+ def update_config(new_config = {})
56
+ new_config = prepare_config(new_config)
57
+ @config = new_config
58
+ @full_name = nil
59
+ @logger = nil
60
+
61
+ debug "updating config to: #{@config.inspect}"
62
+
63
+ remove_triggers
64
+ add_triggers
65
+
66
+ if up?
67
+ # rebuild checks for this process
68
+ remove_watchers
69
+ remove_children
70
+
71
+ add_watchers
72
+ add_children
73
+ end
74
+ end
75
+
76
+ # is pid_file under Eye::Process control, or not
77
+ def control_pid?
78
+ !!self[:daemonize]
79
+ end
80
+
81
+ end
@@ -0,0 +1,73 @@
1
+ module Eye::Process::Controller
2
+
3
+ def send_command(command, *args)
4
+ schedule command, *args, Eye::Reason::User.new(command)
5
+ end
6
+
7
+ def start
8
+ res = if set_pid_from_file
9
+ if process_really_running?
10
+ info "process <#{self.pid}> from pid_file is already running"
11
+ switch :already_running
12
+ :ok
13
+ else
14
+ info "pid_file found, but process <#{self.pid}> is down, starting..."
15
+ start_process
16
+ end
17
+ else
18
+ info 'pid_file not found, starting...'
19
+ start_process
20
+ end
21
+
22
+ res
23
+ end
24
+
25
+ def stop
26
+ stop_process
27
+ switch :unmonitoring
28
+ end
29
+
30
+ def restart
31
+ unless pid # unmonitored case
32
+ try_update_pid_from_file
33
+ end
34
+
35
+ restart_process
36
+ end
37
+
38
+ def monitor
39
+ if self[:auto_start]
40
+ start
41
+ else
42
+ if try_update_pid_from_file
43
+ info "process <#{self.pid}> from pid_file is already running"
44
+ switch :already_running
45
+ else
46
+ warn 'process not found, unmonitoring'
47
+ schedule :unmonitor, Eye::Reason.new(:'not found')
48
+ end
49
+ end
50
+ end
51
+
52
+ def unmonitor
53
+ switch :unmonitoring
54
+ end
55
+
56
+ def delete
57
+ if self[:stop_on_delete]
58
+ info 'process has stop_on_delete option, so sync-stop it first'
59
+ stop
60
+ end
61
+
62
+ remove_watchers
63
+ remove_children
64
+ remove_triggers
65
+
66
+ terminate
67
+ end
68
+
69
+ def signal(sig = 0)
70
+ send_signal(sig) if self.pid
71
+ end
72
+
73
+ end
@@ -0,0 +1,78 @@
1
+ module Eye::Process::Data
2
+
3
+ def logger_tag
4
+ full_name
5
+ end
6
+
7
+ def app_name
8
+ self[:application]
9
+ end
10
+
11
+ def group_name
12
+ (self[:group] == '__default__') ? nil : self[:group]
13
+ end
14
+
15
+ def group_name_pure
16
+ self[:group]
17
+ end
18
+
19
+ def full_name
20
+ @full_name ||= [app_name, group_name, self[:name]].compact.join(':')
21
+ end
22
+
23
+ def status_data(debug = false)
24
+ p_st = self_status_data(debug)
25
+
26
+ if children.present?
27
+ p_st.merge(:subtree => Eye::Utils::AliveArray.new(children.values).map{|c| c.status_data(debug) } )
28
+ elsif self[:monitor_children] && self.up?
29
+ p_st.merge(:subtree => [{name: '=loading children='}])
30
+ else
31
+ # common state
32
+ p_st
33
+ end
34
+ end
35
+
36
+ def self_status_data(debug = false)
37
+ h = { name: name, state: state,
38
+ type: (self.class == Eye::ChildProcess ? :child_process : :process),
39
+ resources: Eye::SystemResources.resources(pid) }
40
+
41
+ if @states_history
42
+ h.merge!( state_changed_at: @states_history.last_state_changed_at.to_i,
43
+ state_reason: @states_history.last_reason )
44
+ end
45
+
46
+ h.merge!(debug: debug_data) if debug
47
+ h.merge!(current_command: current_scheduled_command) if current_scheduled_command
48
+
49
+ h
50
+ end
51
+
52
+ def debug_data
53
+ { queue: scheduler_actions_list, watchers: @watchers.keys }
54
+ end
55
+
56
+ def sub_object?(obj)
57
+ return false if self.class == Eye::ChildProcess
58
+ self.children.each { |_, child| return true if child == obj }
59
+ false
60
+ end
61
+
62
+ def environment_string
63
+ s = []
64
+ @config[:environment].each { |k, v| s << "#{k}=#{v}" }
65
+ s * ' '
66
+ end
67
+
68
+ def shell_string(dir = true)
69
+ str = ''
70
+ str += "cd #{self[:working_dir]} && " if dir
71
+ str += environment_string
72
+ str += ' '
73
+ str += self[:start_command]
74
+ str += ' &' if self[:daemonize]
75
+ str
76
+ end
77
+
78
+ end