eye 0.8.celluloid15 → 0.8.pre

data/lib/eye/notify/slack.rb

@@ -8,8 +8,8 @@ class Eye::Notify::Slack < Eye::Notify
   # end
 
   param :webhook_url, String, true
-  param :channel,     String, nil, '#default'
-  param :username,    String, nil, 'eye'
+  param :channel,     String, nil, "#default"
+  param :username,    String, nil, "eye"
 
   param :icon, String
 
@@ -35,5 +35,4 @@ class Eye::Notify::Slack < Eye::Notify
     payload << "> #{msg_message}"
     payload
   end
-
-end
+end

data/lib/eye/process.rb

@@ -1,7 +1,6 @@
 require 'celluloid'
 
 class Eye::Process
-
   include Celluloid
 
   autoload :Config,           'eye/process/config'
@@ -78,7 +77,6 @@ class Eye::Process
 
   # validate
   extend Eye::Process::Validate
-
 end
 
 # include state_machine states

data/lib/eye/process/children.rb

@@ -31,17 +31,17 @@ module Eye::Process::Children
 
     if new_children.present?
       new_children.each do |child_pid|
-        cfg = self[:monitor_children].try :update, notify: self[:notify]
+        cfg = self[:monitor_children].try :update, :notify => self[:notify]
         self.children[child_pid] = Eye::ChildProcess.new(child_pid, cfg, logger.prefix, current_actor)
       end
     end
 
     if removed_children.present?
-      removed_children.each { |child_pid| remove_child(child_pid) }
+      removed_children.each{|child_pid| remove_child(child_pid) }
     end
 
-    h = { new: new_children.size, removed: removed_children.size, exists: exist_children.size }
-    debug { "children info: #{h.inspect}" }
+    h = {:new => new_children.size, :removed => removed_children.size, :exists => exist_children.size }
+    debug { "children info: #{ h.inspect }" }
 
     @updating_children = false
     h

data/lib/eye/process/commands.rb

@@ -25,6 +25,7 @@ module Eye::Process::Commands
         sleep 0.2 # little grace
       end
 
+      self.pid = nil
       switch :crashed
     end
 
@@ -41,8 +42,6 @@ module Eye::Process::Commands
 
     switch :stopping
 
-    return unless check_identity
-
     kill_process
 
     if process_really_running?
@@ -70,11 +69,9 @@ module Eye::Process::Commands
     switch :restarting
 
     if self[:restart_command]
-      return unless check_identity
-      if execute_restart_command
-        sleep_grace(:restart_grace)
-      end
-      result = process_really_running? || (load_external_pid_file == :ok)
+      execute_restart_command
+      sleep_grace(:restart_grace)
+      result = check_alive_with_refresh_pid_if_needed
       switch(result ? :restarted : :crashed)
     else
       stop_process
@@ -99,9 +96,10 @@ private
     if self[:stop_command]
       cmd = prepare_command(self[:stop_command])
       info "executing: `#{cmd}` with stop_timeout: #{self[:stop_timeout].to_f}s and stop_grace: #{self[:stop_grace].to_f}s"
-      res = execute(cmd, config.merge(timeout: self[:stop_timeout]))
+      res = execute(cmd, config.merge(:timeout => self[:stop_timeout]))
 
       if res[:error]
+
         if res[:error].class == Timeout::Error
           error "stop_command failed with #{res[:error].inspect}; try tuning the stop_timeout value"
         else
@@ -114,6 +112,7 @@ private
     elsif self[:stop_signals]
       info "executing stop_signals #{self[:stop_signals].inspect}"
       stop_signals = self[:stop_signals].clone
+
       signal = stop_signals.shift
       send_signal(signal)
 
@@ -121,7 +120,7 @@ private
         delay = stop_signals.shift
         signal = stop_signals.shift
 
-        if wait_for_condition(delay.to_f, 0.3) { !process_really_running? }
+        if wait_for_condition(delay.to_f, 0.3){ !process_really_running? }
           info 'has terminated'
           break
         end
@@ -134,6 +133,7 @@ private
     else # default command
       debug { "executing: `kill -TERM #{self.pid}` with stop_grace: #{self[:stop_grace].to_f}s" }
       send_signal(:TERM)
+
       sleep_grace(:stop_grace)
 
       # if process not die here, by default we force kill it
@@ -154,7 +154,7 @@ private
     cmd = prepare_command(self[:restart_command])
     info "executing: `#{cmd}` with restart_timeout: #{self[:restart_timeout].to_f}s and restart_grace: #{self[:restart_grace].to_f}s"
 
-    res = execute(cmd, config.merge(timeout: self[:restart_timeout]))
+    res = execute(cmd, config.merge(:timeout => self[:restart_timeout]))
 
     if res[:error]
 
@@ -169,9 +169,11 @@ private
   end
 
   def daemonize_process
+    time_before = Time.now
     res = daemonize(self[:start_command], config)
-    info "daemonizing: `#{self[:start_command]}` with start_grace: #{self[:start_grace].to_f}s, env: '#{environment_string}'}" \
-      ", <#{res[:pid]}> (in #{self[:working_dir]})"
+    start_time = Time.now - time_before
+
+    info "daemonizing: `#{self[:start_command]}` with start_grace: #{self[:start_grace].to_f}s, env: '#{environment_string}', <#{res[:pid]}> (in #{self[:working_dir]})"
 
     if res[:error]
 
@@ -181,21 +183,21 @@ private
         error "daemonize failed with #{res[:error].inspect}"
       end
 
-      return { error: res[:error].inspect }
+      return {:error => res[:error].inspect}
     end
 
     self.pid = res[:pid]
 
     unless self.pid
       error 'no pid was returned'
-      return { error: :empty_pid }
+      return {:error => :empty_pid}
     end
 
     sleep_grace(:start_grace)
 
     unless process_really_running?
       error "process <#{self.pid}> not found, it may have crashed (#{check_logs_str})"
-      return { error: :not_really_running }
+      return {:error => :not_really_running}
     end
 
     # if we using leaf child stratedy, pid should be used as last child process
@@ -210,46 +212,47 @@ private
     end
 
     if control_pid? && !failsafe_save_pid
-      return { error: :cant_write_pid }
+      return {:error => :cant_write_pid}
     end
 
     res
   end
 
   def execute_process
-    info "executing: `#{self[:start_command]}` with start_timeout: #{config[:start_timeout].to_f}s" \
-      ", start_grace: #{self[:start_grace].to_f}s, env: '#{environment_string}' (in #{self[:working_dir]})"
-    res = execute(self[:start_command], config.merge(timeout: config[:start_timeout]))
+    info "executing: `#{self[:start_command]}` with start_timeout: #{config[:start_timeout].to_f}s, start_grace: #{self[:start_grace].to_f}s, env: '#{environment_string}' (in #{self[:working_dir]})"
+    time_before = Time.now
+
+    res = execute(self[:start_command], config.merge(:timeout => config[:start_timeout]))
+    start_time = Time.now - time_before
 
     if res[:error]
 
       if res[:error].message == 'Permission denied - open'
         error "execution failed with #{res[:error].inspect}; ensure that #{[self[:stdout], self[:stderr]]} are writable"
       elsif res[:error].class == Timeout::Error
-        error "execution failed with #{res[:error].inspect}; try increasing the start_timeout value" \
-          "(the current value of #{self[:start_timeout]}s seems too short)"
+        error "execution failed with #{res[:error].inspect}; try increasing the start_timeout value (the current value of #{self[:start_timeout]}s seems too short)"
       else
         error "execution failed with #{res[:error].inspect}"
       end
 
-      return { error: res[:error].inspect }
+      return {:error => res[:error].inspect}
     end
 
     sleep_grace(:start_grace)
 
-    case load_external_pid_file
-      when :ok
-        res.merge(pid: self.pid)
-      when :no_pid_file
-        error "exit status #{res[:exitstatus]}, pid_file (#{self[:pid_file_ex]}) did not appear within the " \
-          "start_grace period (#{self[:start_grace].to_f}s); check your start_command, or tune the start_grace " \
-          'value (eye expect process to create pid_file in self-daemonization mode)'
-        { error: :pid_not_found }
-      when :not_running
-        error "exit status #{res[:exitstatus]}, process <#{@last_loaded_pid}> (from #{self[:pid_file_ex]}) was not found; " \
-          "ensure that the pid_file is being updated correctly (#{check_logs_str})"
-        { error: :not_really_running }
+    unless set_pid_from_file
+      error "exit status #{res[:exitstatus]}, pid_file (#{self[:pid_file_ex]}) did not appear within the start_grace period (#{self[:start_grace].to_f}s); check your start_command, or tune the start_grace value (eye expect process to create pid_file in self-daemonization mode)"
+      return {:error => :pid_not_found}
     end
+
+    unless process_really_running?
+      error "exit status #{res[:exitstatus]}, process <#{self.pid}> (from #{self[:pid_file_ex]}) was not found; ensure that the pid_file is being updated correctly (#{check_logs_str})"
+      return {:error => :not_really_running}
+    end
+
+    res[:pid] = self.pid
+    info "exit status #{res[:exitstatus]}, process <#{res[:pid]}> (from #{self[:pid_file_ex]}) was found"
+    res
   end
 
   def check_logs_str
@@ -275,14 +278,12 @@ private
   end
 
   def execute_user_command(name, cmd)
-    return unless check_identity
-
     info "executing user command #{name} #{cmd.inspect}"
 
     # cmd is string, or array of signals
     if cmd.is_a?(String)
       cmd = prepare_command(cmd)
-      res = execute(cmd, config.merge(timeout: 120))
+      res = execute(cmd, config.merge(:timeout => 120))
       error "cmd #{cmd} error #{res.inspect}" if res[:error]
     elsif cmd.is_a?(Array)
       signals = cmd.clone
@@ -292,7 +293,7 @@ private
       while signals.present?
         delay = signals.shift
         signal = signals.shift
-        if wait_for_condition(delay.to_f, 0.3) { !process_really_running? }
+        if wait_for_condition(delay.to_f, 0.3){ !process_really_running? }
           info 'has terminated'
           break
         end

data/lib/eye/process/config.rb

@@ -1,29 +1,25 @@
 module Eye::Process::Config
 
   DEFAULTS = {
-    keep_alive: true, # restart when crashed
-    check_alive_period: 5.seconds,
+    :keep_alive => true, # restart when crashed
+    :check_alive_period => 5.seconds,
 
-    check_identity: true,
-    check_identity_period: 60.seconds,
-    check_identity_grace: 60.seconds,
+    :start_timeout => 15.seconds,
+    :stop_timeout => 10.seconds,
+    :restart_timeout => 10.seconds,
 
-    start_timeout: 15.seconds,
-    stop_timeout: 10.seconds,
-    restart_timeout: 10.seconds,
+    :start_grace => 2.5.seconds,
+    :stop_grace => 0.5.seconds,
+    :restart_grace => 1.second,
 
-    start_grace: 2.5.seconds,
-    stop_grace: 0.5.seconds,
-    restart_grace: 1.second,
+    :daemonize => false,
+    :auto_start => true, # auto start on monitor action
 
-    daemonize: false,
-    auto_start: true, # auto start on monitor action
+    :children_update_period => 30.seconds,
+    :clear_pid => true, # by default clear pid on stop
 
-    children_update_period: 30.seconds,
-    clear_pid: true, # by default clear pid on stop
-
-    auto_update_pidfile_grace: 30.seconds,
-    revert_fuckup_pidfile_grace: 120.seconds
+    :auto_update_pidfile_grace => 30.seconds,
+    :revert_fuckup_pidfile_grace => 120.seconds,
   }
 
   def prepare_config(new_config)
@@ -31,14 +27,12 @@ module Eye::Process::Config
     h[:pid_file_ex] = Eye::System.normalized_file(h[:pid_file], h[:working_dir]) if h[:pid_file]
     h[:checks] = {} if h[:checks].blank?
     h[:triggers] = {} if h[:triggers].blank?
-    if upd = h.try(:[], :monitor_children).try(:[], :children_update_period)
-      h[:children_update_period] = upd
-    end
+    h[:children_update_period] = h[:monitor_children][:children_update_period] if h[:monitor_children] && h[:monitor_children][:children_update_period]
 
     # check speedy flapping by default
     if h[:triggers].blank? || !h[:triggers][:flapping]
       h[:triggers] ||= {}
-      h[:triggers][:flapping] = { type: :flapping, times: 10, within: 10.seconds }
+      h[:triggers][:flapping] = {:type => :flapping, :times => 10, :within => 10.seconds}
     end
 
     h[:stdout] = Eye::System.normalized_file(h[:stdout], h[:working_dir]) if h[:stdout]

data/lib/eye/process/controller.rb

@@ -5,12 +5,21 @@ module Eye::Process::Controller
   end
 
   def start
-    if load_external_pid_file == :ok
-      switch :already_running
-      :ok
+    res = if set_pid_from_file
+      if process_really_running?
+        info "process <#{self.pid}> from pid_file is already running"
+        switch :already_running
+        :ok
+      else
+        info "pid_file found, but process <#{self.pid}> is down, starting..."
+        start_process
+      end
     else
+      info 'pid_file not found, starting...'
       start_process
     end
+
+    res
   end
 
   def stop
@@ -19,7 +28,10 @@ module Eye::Process::Controller
   end
 
   def restart
-    load_external_pid_file unless pid # unmonitored case
+    unless pid # unmonitored case
+      try_update_pid_from_file
+    end
+
     restart_process
   end
 
@@ -27,9 +39,11 @@ module Eye::Process::Controller
     if self[:auto_start]
       start
     else
-      if load_external_pid_file == :ok
+      if try_update_pid_from_file
+        info "process <#{self.pid}> from pid_file is already running"
         switch :already_running
       else
+        warn 'process not found, unmonitoring'
         schedule :unmonitor, Eye::Reason.new(:'not found')
       end
     end

data/lib/eye/process/data.rb

@@ -20,32 +20,30 @@ module Eye::Process::Data
     @full_name ||= [app_name, group_name, self[:name]].compact.join(':')
   end
 
-  def status_data(opts = {})
-    p_st = self_status_data(opts)
+  def status_data(debug = false)
+    p_st = self_status_data(debug)
 
     if children.present?
-      p_st.merge(subtree: Eye::Utils::AliveArray.new(children.values).map { |c| c.status_data(opts) })
+      p_st.merge(:subtree => Eye::Utils::AliveArray.new(children.values).map{|c| c.status_data(debug) } )
     elsif self[:monitor_children] && self.up?
-      p_st.merge(subtree: [{ name: '=loading children=' }])
+      p_st.merge(:subtree => [{name: '=loading children='}])
     else
       # common state
       p_st
     end
   end
 
-  def self_status_data(opts)
-    h = { name: name,
-          state: state,
+  def self_status_data(debug = false)
+    h = { name: name, state: state,
           type: (self.class == Eye::ChildProcess ? :child_process : :process),
           resources: Eye::SystemResources.resources(pid) }
 
     if @states_history
-      h[:state_changed_at] = @states_history.last_state_changed_at.to_i
-      h[:state_reason] = @states_history.last_reason.to_s
+      h.merge!( state_changed_at: @states_history.last_state_changed_at.to_i,
+                state_reason: @states_history.last_reason.to_s )
     end
 
-    h[:debug] = debug_data if opts[:debug]
-    h[:procline] = Eye::SystemResources.args(self.pid) if opts[:procline]
+    h[:debug] = debug_data if debug
     h[:current_command] = current_scheduled_command if current_scheduled_command
 
     h

data/lib/eye/process/monitor.rb

@@ -2,112 +2,102 @@ module Eye::Process::Monitor
 
 private
 
-  def load_external_pid_file
-    newpid = failsafe_load_pid
-
-    if !newpid
-      self.pid = nil
-      info 'load_external_pid_file: pid_file not found'
-      :no_pid_file
-    elsif process_pid_running?(newpid)
-      self.pid = newpid
-      res = compare_identity
-      if res == :fail
-        warn "load_external_pid_file: process <#{self.pid}> from pid_file failed check_identity"
-        :bad_identity
-      else
-        args = Eye::SystemResources.args(self.pid)
-        info "load_external_pid_file: process <#{self.pid}> from pid_file found and running (identity: #{res}) (#{args})"
-        :ok
-      end
-    else
-      @last_loaded_pid = newpid
-      self.pid = nil
-      info "load_external_pid_file: pid_file found, but process <#{newpid}> not found"
-      :not_running
-    end
-  end
-
-  def check_alive
-    return unless up?
-
-    # check that process runned
+  def check_alive_with_refresh_pid_if_needed
     if process_really_running?
-      check_pid_file
-    else
-      warn "check_alive: process <#{self.pid}> not found"
-      notify :info, 'crashed!'
-      clear_pid_file(true) if control_pid?
+      return true
 
-      switch :crashed, Eye::Reason.new(:crashed)
+    else
+      warn 'process not really running'
+      try_update_pid_from_file
     end
   end
 
-  def check_pid_file
-    ppid = failsafe_load_pid
-    return if ppid == self.pid
+  def try_update_pid_from_file
+    # if pid file was rewritten
+    newpid = load_pid_from_file
+    if newpid != self.pid
+      info "process <#{self.pid}> changed pid to <#{newpid}>, updating..." if self.pid
+      self.pid = newpid
 
-    msg = "check_alive: pid_file (#{self[:pid_file]}) changed by itself (<#{self.pid}> => <#{ppid}>)"
-    if control_pid?
-      msg += ", reverting to <#{self.pid}> (the pid_file is controlled by eye)"
-      unless failsafe_save_pid
-        msg += ', pid_file write failed! O_o'
+      if process_really_running?
+        return true
+      else
+        warn "process <#{newpid}> was not found"
+        return false
       end
     else
-      changed_ago_s = Time.now - pid_file_ctime
-
-      if !ppid
-        msg += ", reverting to <#{self.pid}> (the pid_file is empty)"
-        unless failsafe_save_pid
-          msg += ', pid_file write failed! O_o'
-        end
+      debug { 'process was not found' }
+      return false
+    end
+  end
 
-      elsif (changed_ago_s > self[:auto_update_pidfile_grace]) && process_pid_running?(ppid) && (compare_identity(ppid) != :fail)
-        msg += ", trusting this change, and now monitor <#{ppid}>"
-        self.pid = ppid
+  def check_alive
+    if up?
 
-      elsif changed_ago_s > self[:revert_fuckup_pidfile_grace]
-        msg += " over #{self[:revert_fuckup_pidfile_grace]}s ago, reverting to <#{self.pid}>, because <#{ppid}> not alive"
-        unless failsafe_save_pid
-          msg += ', pid_file write failed! O_o'
-        end
+      # check that process runned
+      unless process_really_running?
+        warn "check_alive: process <#{self.pid}> not found"
+        notify :info, 'crashed!'
+        clear_pid_file if control_pid? && self.pid && load_pid_from_file == self.pid
 
+        switch :crashed, Eye::Reason.new(:crashed)
       else
-        msg += ', ignoring self-managed pid change'
+        # check that pid_file still here
+        ppid = failsafe_load_pid
+
+        if ppid != self.pid
+          msg = "check_alive: pid_file (#{self[:pid_file]}) changed by itself (<#{self.pid}> => <#{ppid}>)"
+          if control_pid?
+            msg += ", reverting to <#{self.pid}> (the pid_file is controlled by eye)"
+            unless failsafe_save_pid
+              msg += ", pid_file write failed! O_o"
+            end
+          else
+            changed_ago_s = Time.now - pid_file_ctime
+
+            if ppid == nil
+              msg += ", reverting to <#{self.pid}> (the pid_file is empty)"
+              unless failsafe_save_pid
+                msg += ", pid_file write failed! O_o"
+              end
+
+            elsif (changed_ago_s > self[:auto_update_pidfile_grace]) && process_pid_running?(ppid)
+              msg += ", trusting this change, and now monitor <#{ppid}>"
+              self.pid = ppid
+
+            elsif (changed_ago_s > self[:revert_fuckup_pidfile_grace])
+              msg += " over #{self[:revert_fuckup_pidfile_grace]}s ago, reverting to <#{self.pid}>, because <#{ppid}> not alive"
+              unless failsafe_save_pid
+                msg += ", pid_file write failed! O_o"
+              end
+
+            else
+              msg += ', ignoring self-managed pid change'
+            end
+          end
+
+          warn msg
+        end
       end
     end
-
-    warn msg
-  end
-
-  def check_identity
-    if compare_identity == :fail
-      notify :info, 'crashed by identity!'
-      switch :crashed, Eye::Reason.new(:crashed_by_identity)
-      clear_pid_file if self[:clear_pid]
-      false
-    else
-      true
-    end
   end
 
   def check_crash
-    unless down?
-      debug { 'check crashed: skipped, process is not in down' }
-      return
-    end
-
-    if self[:keep_alive]
-      warn 'check crashed: process is down'
-
-      if self[:restore_in]
-        schedule_in self[:restore_in].to_f, :restore, Eye::Reason.new(:crashed)
+    if down?
+      if self[:keep_alive]
+        warn 'check crashed: process is down'
+
+        if self[:restore_in]
+          schedule_in self[:restore_in].to_f, :restore, Eye::Reason.new(:crashed)
+        else
+          schedule :restore, Eye::Reason.new(:crashed)
+        end
       else
-        schedule :restore, Eye::Reason.new(:crashed)
+        warn 'check crashed: process without keep_alive'
+        schedule :unmonitor, Eye::Reason.new(:crashed)
       end
     else
-      warn 'check crashed: process without keep_alive'
-      schedule :unmonitor, Eye::Reason.new(:crashed)
+      debug { 'check crashed: skipped, process is not in down' }
     end
   end