eye 0.7 → 0.8.celluloid15

data/lib/eye/process/data.rb

@@ -20,30 +20,32 @@ module Eye::Process::Data
     @full_name ||= [app_name, group_name, self[:name]].compact.join(':')
   end
 
-  def status_data(debug = false)
-    p_st = self_status_data(debug)
+  def status_data(opts = {})
+    p_st = self_status_data(opts)
 
     if children.present?
-      p_st.merge(:subtree => Eye::Utils::AliveArray.new(children.values).map{|c| c.status_data(debug) } )
+      p_st.merge(subtree: Eye::Utils::AliveArray.new(children.values).map { |c| c.status_data(opts) })
     elsif self[:monitor_children] && self.up?
-      p_st.merge(:subtree => [{name: '=loading children='}])
+      p_st.merge(subtree: [{ name: '=loading children=' }])
     else
       # common state
       p_st
     end
   end
 
-  def self_status_data(debug = false)
-    h = { name: name, state: state,
+  def self_status_data(opts)
+    h = { name: name,
+          state: state,
           type: (self.class == Eye::ChildProcess ? :child_process : :process),
           resources: Eye::SystemResources.resources(pid) }
 
     if @states_history
-      h.merge!( state_changed_at: @states_history.last_state_changed_at.to_i,
-                state_reason: @states_history.last_reason.to_s )
+      h[:state_changed_at] = @states_history.last_state_changed_at.to_i
+      h[:state_reason] = @states_history.last_reason.to_s
     end
 
-    h[:debug] = debug_data if debug
+    h[:debug] = debug_data if opts[:debug]
+    h[:procline] = Eye::SystemResources.args(self.pid) if opts[:procline]
     h[:current_command] = current_scheduled_command if current_scheduled_command
 
     h

data/lib/eye/process/monitor.rb

@@ -2,102 +2,112 @@ module Eye::Process::Monitor
 
 private
 
-  def check_alive_with_refresh_pid_if_needed
-    if process_really_running?
-      return true
-
+  def load_external_pid_file
+    newpid = failsafe_load_pid
+
+    if !newpid
+      self.pid = nil
+      info 'load_external_pid_file: pid_file not found'
+      :no_pid_file
+    elsif process_pid_running?(newpid)
+      self.pid = newpid
+      res = compare_identity
+      if res == :fail
+        warn "load_external_pid_file: process <#{self.pid}> from pid_file failed check_identity"
+        :bad_identity
+      else
+        args = Eye::SystemResources.args(self.pid)
+        info "load_external_pid_file: process <#{self.pid}> from pid_file found and running (identity: #{res}) (#{args})"
+        :ok
+      end
     else
-      warn 'process not really running'
-      try_update_pid_from_file
+      @last_loaded_pid = newpid
+      self.pid = nil
+      info "load_external_pid_file: pid_file found, but process <#{newpid}> not found"
+      :not_running
     end
   end
 
-  def try_update_pid_from_file
-    # if pid file was rewritten
-    newpid = load_pid_from_file
-    if newpid != self.pid
-      info "process <#{self.pid}> changed pid to <#{newpid}>, updating..." if self.pid
-      self.pid = newpid
+  def check_alive
+    return unless up?
 
-      if process_really_running?
-        return true
-      else
-        warn "process <#{newpid}> was not found"
-        return false
-      end
+    # check that process runned
+    if process_really_running?
+      check_pid_file
     else
-      debug { 'process was not found' }
-      return false
+      warn "check_alive: process <#{self.pid}> not found"
+      notify :info, 'crashed!'
+      clear_pid_file(true) if control_pid?
+
+      switch :crashed, Eye::Reason.new(:crashed)
     end
   end
 
-  def check_alive
-    if up?
+  def check_pid_file
+    ppid = failsafe_load_pid
+    return if ppid == self.pid
 
-      # check that process runned
-      unless process_really_running?
-        warn "check_alive: process <#{self.pid}> not found"
-        notify :info, 'crashed!'
-        clear_pid_file if control_pid? && self.pid && load_pid_from_file == self.pid
+    msg = "check_alive: pid_file (#{self[:pid_file]}) changed by itself (<#{self.pid}> => <#{ppid}>)"
+    if control_pid?
+      msg += ", reverting to <#{self.pid}> (the pid_file is controlled by eye)"
+      unless failsafe_save_pid
+        msg += ', pid_file write failed! O_o'
+      end
+    else
+      changed_ago_s = Time.now - pid_file_ctime
 
-        switch :crashed, Eye::Reason.new(:crashed)
-      else
-        # check that pid_file still here
-        ppid = failsafe_load_pid
-
-        if ppid != self.pid
-          msg = "check_alive: pid_file (#{self[:pid_file]}) changed by itself (<#{self.pid}> => <#{ppid}>)"
-          if control_pid?
-            msg += ", reverting to <#{self.pid}> (the pid_file is controlled by eye)"
-            unless failsafe_save_pid
-              msg += ", pid_file write failed! O_o"
-            end
-          else
-            changed_ago_s = Time.now - pid_file_ctime
-
-            if ppid == nil
-              msg += ", reverting to <#{self.pid}> (the pid_file is empty)"
-              unless failsafe_save_pid
-                msg += ", pid_file write failed! O_o"
-              end
-
-            elsif (changed_ago_s > self[:auto_update_pidfile_grace]) && process_pid_running?(ppid)
-              msg += ", trusting this change, and now monitor <#{ppid}>"
-              self.pid = ppid
-
-            elsif (changed_ago_s > self[:revert_fuckup_pidfile_grace])
-              msg += " over #{self[:revert_fuckup_pidfile_grace]}s ago, reverting to <#{self.pid}>, because <#{ppid}> not alive"
-              unless failsafe_save_pid
-                msg += ", pid_file write failed! O_o"
-              end
-
-            else
-              msg += ', ignoring self-managed pid change'
-            end
-          end
-
-          warn msg
+      if !ppid
+        msg += ", reverting to <#{self.pid}> (the pid_file is empty)"
+        unless failsafe_save_pid
+          msg += ', pid_file write failed! O_o'
         end
+
+      elsif (changed_ago_s > self[:auto_update_pidfile_grace]) && process_pid_running?(ppid) && (compare_identity(ppid) != :fail)
+        msg += ", trusting this change, and now monitor <#{ppid}>"
+        self.pid = ppid
+
+      elsif changed_ago_s > self[:revert_fuckup_pidfile_grace]
+        msg += " over #{self[:revert_fuckup_pidfile_grace]}s ago, reverting to <#{self.pid}>, because <#{ppid}> not alive"
+        unless failsafe_save_pid
+          msg += ', pid_file write failed! O_o'
+        end
+
+      else
+        msg += ', ignoring self-managed pid change'
       end
     end
+
+    warn msg
+  end
+
+  def check_identity
+    if compare_identity == :fail
+      notify :info, 'crashed by identity!'
+      switch :crashed, Eye::Reason.new(:crashed_by_identity)
+      clear_pid_file if self[:clear_pid]
+      false
+    else
+      true
+    end
   end
 
   def check_crash
-    if down?
-      if self[:keep_alive]
-        warn 'check crashed: process is down'
-
-        if self[:restore_in]
-          schedule_in self[:restore_in].to_f, :restore, Eye::Reason.new(:crashed)
-        else
-          schedule :restore, Eye::Reason.new(:crashed)
-        end
+    unless down?
+      debug { 'check crashed: skipped, process is not in down' }
+      return
+    end
+
+    if self[:keep_alive]
+      warn 'check crashed: process is down'
+
+      if self[:restore_in]
+        schedule_in self[:restore_in].to_f, :restore, Eye::Reason.new(:crashed)
       else
-        warn 'check crashed: process without keep_alive'
-        schedule :unmonitor, Eye::Reason.new(:crashed)
+        schedule :restore, Eye::Reason.new(:crashed)
       end
     else
-      debug { 'check crashed: skipped, process is not in down' }
+      warn 'check crashed: process without keep_alive'
+      schedule :unmonitor, Eye::Reason.new(:crashed)
     end
   end
 

data/lib/eye/process/notify.rb

@@ -5,21 +5,21 @@ module Eye::Process::Notify
   # 2) checker bounded to restart process [:warn]
   # 3) flapping + switch to unmonitored [:error]
 
-  LEVELS = {:debug => 0, :info => 1, :warn => 2, :error => 3, :fatal => 4}
+  LEVELS = { debug: 0, info: 1, warn: 2, error: 3, fatal: 4 }
 
   def notify(level, msg)
     # logging it
     error "NOTIFY: #{msg}" if ilevel(level) > ilevel(:info)
 
+    return if self[:notify].blank?
+
     # send notifies
-    if self[:notify].present?
-      message = {:message => msg, :name => name,
-        :full_name => full_name, :pid => pid, :host => Eye::Local.host, :level => level,
-        :at => Time.now }
-
-      self[:notify].each do |contact, not_level|
-        Eye::Notify.notify(contact, message) if ilevel(level) >= ilevel(not_level)
-      end
+    message = { message: msg, name: name,
+                full_name: full_name, pid: pid, host: Eye::Local.host, level: level,
+                at: Time.now }
+
+    self[:notify].each do |contact, not_level|
+      Eye::Notify.notify(contact, message) if ilevel(level) >= ilevel(not_level)
     end
   end
 
@@ -29,4 +29,4 @@ private
     LEVELS[level].to_i
   end
 
-end
+end

data/lib/eye/process/scheduler.rb

@@ -2,32 +2,30 @@ module Eye::Process::Scheduler
 
   # ex: schedule :update_config, config, "reason: update_config"
   def schedule(command, *args, &block)
-    if scheduler.alive?
-      if scheduler_freeze?
-        warn ":#{command} ignoring to schedule, because scheduler is freeze"
-        return
-      end
-
-      unless self.respond_to?(command, true)
-        warn ":#{command} scheduling is unsupported"
-        return
-      end
-
-      reason = if args.present? && args[-1].kind_of?(Eye::Reason)
-        args.pop
-      end
-
-      info "schedule :#{command} #{reason ? "(reason: #{reason})" : nil}"
-
-      if reason.class == Eye::Reason
-        # for auto reasons
-        # skip already running commands and all in chain
-        scheduler.add_wo_dups_current(:scheduled_action, command, {:args => args, :reason => reason}, &block)
-      else
-        # for manual, or without reason
-        # skip only for last in chain
-        scheduler.add_wo_dups(:scheduled_action, command, {:args => args, :reason => reason}, &block)
-      end
+    return unless scheduler.alive?
+
+    if scheduler_freeze?
+      warn ":#{command} ignoring to schedule, because scheduler is freeze"
+      return
+    end
+
+    unless self.respond_to?(command, true)
+      warn ":#{command} scheduling is unsupported"
+      return
+    end
+
+    reason = args.pop if args.present? && args[-1].is_a?(Eye::Reason)
+
+    info "schedule :#{command} #{reason ? "(reason: #{reason})" : nil}"
+
+    if reason.class == Eye::Reason
+      # for auto reasons
+      # skip already running commands and all in chain
+      scheduler.add_wo_dups_current(:scheduled_action, command, args: args, reason: reason, block: block)
+    else
+      # for manual, or without reason
+      # skip only for last in chain
+      scheduler.add_wo_dups(:scheduled_action, command, args: args, reason: reason, block: block)
     end
   end
 
@@ -39,16 +37,16 @@ module Eye::Process::Scheduler
     end
   end
 
-  def scheduled_action(command, h = {}, &block)
-    reason = h.delete(:reason)
-    info "=> #{command} #{h[:args].present? ? "#{h[:args]*',' }" : nil} #{reason ? "(reason: #{reason})" : nil}"
+  def scheduled_action(command, h = {})
+    reason = h[:reason]
+    info "=> #{command} #{h[:args].present? ? "#{h[:args] * ','}" : nil} #{reason ? "(reason: #{reason})" : nil}"
 
     @current_scheduled_command = command
     @last_scheduled_command = command
     @last_scheduled_reason = reason
     @last_scheduled_at = Time.now
 
-    send(command, *h[:args], &block)
+    send(command, *h[:args], &h[:block])
     @current_scheduled_command = nil
     info "<= #{command}"
 
@@ -59,8 +57,14 @@ module Eye::Process::Scheduler
     end
   end
 
+  def execute_proc(*_args, &block)
+    self.instance_exec(&block)
+  rescue Object => ex
+    log_ex(ex)
+  end
+
   def scheduler_actions_list
-    scheduler.list.map{|c| c[:args].first rescue nil }.compact
+    scheduler.list.map { |c| c[:args].first rescue nil }.compact
   end
 
   def scheduler_clear_pending_list
@@ -69,6 +73,7 @@ module Eye::Process::Scheduler
 
   def self.included(base)
     base.finalizer :remove_scheduler
+    base.execute_block_on_receiver :schedule
   end
 
   attr_accessor :current_scheduled_command

data/lib/eye/process/states.rb

@@ -2,6 +2,7 @@ require 'state_machine'
 require 'state_machine/version'
 
 class Eye::Process
+
   class StateError < Exception; end
 
   # do transition
@@ -27,7 +28,7 @@ class Eye::Process
     end
 
     event :crashed do
-      transition [:starting, :restarting, :up] => :down
+      transition [:starting, :restarting, :stopping, :up] => :down
     end
 
     event :stopping do
@@ -59,16 +60,17 @@ class Eye::Process
 
     after_transition any => :unmonitored, :do => :on_unmonitored
 
-    after_transition any-:up => :up, :do => :add_watchers
-    after_transition :up => any-:up, :do => :remove_watchers
+    after_transition any - :up => :up, :do => :add_watchers
+    after_transition :up => any - :up, :do => :remove_watchers
 
-    after_transition any-:up => :up, :do => :add_children
+    after_transition any - :up => :up, :do => :add_children
     after_transition any => [:unmonitored, :down], :do => :remove_children
 
     after_transition :on => :crashed, :do => :on_crashed
   end
 
   def on_crashed
+    self.pid = nil
     schedule :check_crash, Eye::Reason.new(:crashed)
   end
 
@@ -79,7 +81,7 @@ class Eye::Process
   def log_transition(transition)
     if transition.to_name != transition.from_name || @state_reason.is_a?(Eye::Reason::User)
       @states_history.push transition.to_name, @state_reason
-      info "switch :#{transition.event} [:#{transition.from_name} => :#{transition.to_name}] #{@state_reason ? "(reason: #{@state_reason})" : nil}"
+      info "switch :#{transition.event} [:#{transition.from_name} => :#{transition.to_name}] #{"(reason: #{@state_reason})" if @state_reason}"
     end
   end
 

data/lib/eye/process/states_history.rb

@@ -1,17 +1,22 @@
 class Eye::Process::StatesHistory < Eye::Utils::Tail
+
   def push(state, reason = nil, tm = Time.now)
     super(state: state, at: tm.to_i, reason: reason)
   end
 
   def states
-    self.map{|c| c[:state] }
+    self.map { |c| c[:state] }
   end
 
-  def states_for_period(period, from_time = nil)
+  def states_for_period(period, from_time = nil, &block)
     tm = Time.now - period
     tm = [tm, from_time].max if from_time
     tm = tm.to_f
-    self.select{|s| s[:at] >= tm }.map{|c| c[:state] }
+    if block
+      self.each { |s| block.call(s) if s[:at] >= tm }
+    else
+      self.select { |s| s[:at] >= tm }.map { |c| c[:state] }
+    end
   end
 
   def last_state
@@ -25,4 +30,5 @@ class Eye::Process::StatesHistory < Eye::Utils::Tail
   def last_state_changed_at
     Time.at(last[:at])
   end
+
 end

data/lib/eye/process/system.rb

@@ -3,16 +3,7 @@ require 'timeout'
 module Eye::Process::System
 
   def load_pid_from_file
-    res = if File.exist?(self[:pid_file_ex])
-      _pid = File.read(self[:pid_file_ex]).to_i
-      _pid > 0 ? _pid : nil
-    end
-
-    res
-  end
-
-  def set_pid_from_file
-    self.pid = load_pid_from_file
+    File.read(self[:pid_file_ex]).to_i rescue nil
   end
 
   def save_pid_to_file
@@ -26,7 +17,8 @@ module Eye::Process::System
     end
   end
 
-  def clear_pid_file
+  def clear_pid_file(check_content = false)
+    return if check_content && self.pid && load_pid_from_file != self.pid
     info "delete pid_file: #{self[:pid_file_ex]}"
     File.unlink(self[:pid_file_ex])
     true
@@ -38,14 +30,38 @@ module Eye::Process::System
     File.ctime(self[:pid_file_ex]) rescue Time.now
   end
 
+  def get_identity
+    File.mtime(self[:pid_file_ex])
+  rescue Errno::ENOENT
+    nil
+  end
+
+  def compare_identity(pid = self.pid)
+    return :ok unless self[:check_identity]
+    return :no_pid unless pid
+    id = get_identity
+    return :no_pid_file unless id
+    st = Eye::SystemResources.start_time(pid)
+    return :no_start_time unless st
+    st1 = st.to_i
+    id1 = id.to_i
+    if (id1 - st1).abs > self[:check_identity_grace]
+      args = Eye::SystemResources.args(pid)
+      msg = "pid_file: '#{Eye::Utils.human_time2(id)}', process: '#{Eye::Utils.human_time2(st)}' (#{args})"
+      res = (id1 < st1) ? :fail : :touched
+      warn "compare_identity: #{res}, #{msg}"
+      res
+    else
+      :ok
+    end
+  end
+
   def process_really_running?
     process_pid_running?(self.pid)
   end
 
   def process_pid_running?(pid)
-    res = Eye::System.check_pid_alive(pid)
-    debug { "process_really_running?: <#{pid}> #{res.inspect}" }
-    !!res[:result]
+    Eye::System.pid_alive?(pid)
   end
 
   def send_signal(code)
@@ -58,7 +74,7 @@ module Eye::Process::System
     res[:result] == :ok
   end
 
-  def wait_for_condition(timeout, step = 0.1, &block)
+  def wait_for_condition(timeout, step = 0.1, &_block)
     res = nil
     sumtime = 0
 
@@ -73,7 +89,7 @@ module Eye::Process::System
   end
 
   def execute(cmd, cfg = {})
-    defer { Eye::System::execute cmd, cfg }.tap do |res|
+    defer { Eye::System.execute cmd, cfg }.tap do |res|
       notify(:debug, "Bad exit status of command #{cmd.inspect}(#{res[:exitstatus].inspect})") if res[:exitstatus] != 0
     end
   end
@@ -82,7 +98,7 @@ module Eye::Process::System
     Eye::System.daemonize(cmd, cfg)
   end
 
-  def execute_sync(cmd, opts = {:timeout => 1.second})
+  def execute_sync(cmd, opts = { timeout: 1.second })
     execute(cmd, self.config.merge(opts)).tap do |res|
       info "execute_sync `#{cmd}` with res: #{res}"
     end
@@ -97,13 +113,12 @@ module Eye::Process::System
   def failsafe_load_pid
     pid = load_pid_from_file
 
-    if !pid
-      # this is can be symlink changed case
+    unless pid # this is can be symlink changed case
       sleep 0.1
       pid = load_pid_from_file
     end
 
-    pid
+    pid if pid && pid > 0
   end
 
   def failsafe_save_pid