ace-eye 0.6.1

data/lib/eye/process/monitor.rb

@@ -0,0 +1,108 @@
+module Eye::Process::Monitor
+
+private
+
+  def check_alive_with_refresh_pid_if_needed
+    if process_really_running?
+      return true
+
+    else
+      warn 'process not really running'
+      try_update_pid_from_file
+    end
+  end
+
+  def try_update_pid_from_file
+    # if pid file was rewritten
+    newpid = load_pid_from_file
+    if newpid != self.pid
+      info "process <#{self.pid}> changed pid to <#{newpid}>, updating..." if self.pid
+      self.pid = newpid
+
+      if process_really_running?
+        return true
+      else
+        warn "process <#{newpid}> was not found"
+        return false
+      end
+    else
+      debug 'process was not found'
+      return false
+    end
+  end
+
+  def check_alive
+    if up?
+
+      # check that process runned
+      unless process_really_running?
+        warn "check_alive: process <#{self.pid}> not found"
+        notify :info, 'crashed!'
+        clear_pid_file if control_pid? && self.pid && load_pid_from_file == self.pid
+
+        switch :crashed, Eye::Reason.new(:crashed)
+      else
+        # check that pid_file still here
+        ppid = failsafe_load_pid
+
+        if ppid != self.pid
+          msg = "check_alive: pid_file (#{self[:pid_file]}) changed by itself (<#{self.pid}> => <#{ppid}>)"
+          if control_pid?
+            msg += ", reverting to <#{self.pid}> (the pid_file is controlled by eye)"
+            unless failsafe_save_pid
+              msg += ", pid_file write failed! O_o"
+            end
+          else
+            changed_ago_s = Time.now - pid_file_ctime
+
+            if ppid == nil
+              msg += ", reverting to <#{self.pid}> (the pid_file is empty)"
+              unless failsafe_save_pid
+                msg += ", pid_file write failed! O_o"
+              end
+
+            elsif (changed_ago_s > self[:auto_update_pidfile_grace]) && process_pid_running?(ppid)
+              msg += ", trusting this change, and now monitor <#{ppid}>"
+              self.pid = ppid
+
+            elsif (changed_ago_s > self[:revert_fuckup_pidfile_grace])
+              msg += " over #{self[:revert_fuckup_pidfile_grace]}s ago, reverting to <#{self.pid}>, because <#{ppid}> not alive"
+              unless failsafe_save_pid
+                msg += ", pid_file write failed! O_o"
+              end
+
+            else
+              msg += ', ignoring self-managed pid change'
+            end
+          end
+
+          warn msg
+        end
+      end
+    end
+  end
+
+  def check_crash
+    if down?
+      if self[:keep_alive]
+        warn 'check crashed: process is down'
+
+        if self[:restore_in]
+          schedule_in self[:restore_in].to_f, :restore, Eye::Reason.new(:crashed)
+        else
+          schedule :restore, Eye::Reason.new(:crashed)
+        end
+      else
+        warn 'check crashed: process without keep_alive'
+        schedule :unmonitor, Eye::Reason.new(:crashed)
+      end
+    else
+      debug 'check crashed: skipped, process is not in down'
+    end
+  end
+
+  def restore
+    start if down?
+  end
+
+end

data/lib/eye/process/notify.rb

@@ -0,0 +1,32 @@
+module Eye::Process::Notify
+
+  # notify to user:
+  # 1) process crashed by itself, and we restart it [:info]
+  # 2) checker bounded to restart process [:warn]
+  # 3) flapping + switch to unmonitored [:error]
+
+  LEVELS = {:debug => 0, :info => 1, :warn => 2, :error => 3, :fatal => 4}
+
+  def notify(level, msg)
+    # logging it
+    error "NOTIFY: #{msg}" if ilevel(level) > ilevel(:info)
+
+    # send notifies
+    if self[:notify].present?
+      message = {:message => msg, :name => name,
+        :full_name => full_name, :pid => pid, :host => Eye::Local.host, :level => level,
+        :at => Time.now }
+
+      self[:notify].each do |contact, not_level|
+        Eye::Notify.notify(contact, message) if ilevel(level) >= ilevel(not_level)
+      end
+    end
+  end
+
+private
+
+  def ilevel(level)
+    LEVELS[level].to_i
+  end
+
+end

data/lib/eye/process/scheduler.rb

@@ -0,0 +1,82 @@
+module Eye::Process::Scheduler
+
+  # ex: schedule :update_config, config, "reason: update_config"
+  def schedule(command, *args, &block)
+    if scheduler.alive?
+      unless self.respond_to?(command, true)
+        warn ":#{command} scheduling is unsupported"
+        return
+      end
+
+      reason = if args.present? && args[-1].kind_of?(Eye::Reason)
+        args.pop
+      end
+
+      info "schedule :#{command} #{reason ? "(reason: #{reason})" : nil}"
+
+      if reason.class == Eye::Reason
+        # for auto reasons
+        # skip already running commands and all in chain
+        scheduler.add_wo_dups_current(:scheduled_action, command, {:args => args, :reason => reason}, &block)
+      else
+        # for manual, or without reason
+        # skip only for last in chain
+        scheduler.add_wo_dups(:scheduled_action, command, {:args => args, :reason => reason}, &block)
+      end
+    end
+  end
+
+  def schedule_in(interval, command, *args, &block)
+    debug "schedule_in #{interval} :#{command} #{args}"
+    after(interval.to_f) do
+      debug "scheduled_in #{interval} :#{command} #{args}"
+      schedule(command, *args, &block)
+    end
+  end
+
+  def scheduled_action(command, h = {}, &block)
+    reason = h.delete(:reason)
+    info "=> #{command} #{h[:args].present? ? "#{h[:args]*',' }" : nil} #{reason ? "(reason: #{reason})" : nil}"
+
+    @current_scheduled_command = command
+    @last_scheduled_command = command
+    @last_scheduled_reason = reason
+    @last_scheduled_at = Time.now
+
+    send(command, *h[:args], &block)
+    @current_scheduled_command = nil
+    info "<= #{command}"
+
+    schedule_history.push(command, reason, @last_scheduled_at.to_i)
+  end
+
+  def scheduler_actions_list
+    scheduler.list.map{|c| c[:args].first rescue nil }.compact
+  end
+
+  def scheduler_clear_pending_list
+    scheduler.clear_pending_list
+  end
+
+  def self.included(base)
+    base.finalizer :remove_scheduler
+  end
+
+  attr_accessor :current_scheduled_command
+  attr_accessor :last_scheduled_command, :last_scheduled_reason, :last_scheduled_at
+
+  def schedule_history
+    @schedule_history ||= Eye::Process::StatesHistory.new(50)
+  end
+
+private
+
+  def remove_scheduler
+    @scheduler.terminate if @scheduler && @scheduler.alive?
+  end
+
+  def scheduler
+    @scheduler ||= Eye::Utils::CelluloidChain.new(current_actor)
+  end
+
+end

data/lib/eye/process/states.rb

@@ -0,0 +1,86 @@
+require 'state_machine'
+require 'state_machine/version'
+
+class Eye::Process
+  class StateError < Exception; end
+
+  # do transition
+  def switch(name, reason = nil)
+    @state_reason = reason || last_scheduled_reason
+    self.send("#{name}!")
+  end
+
+  state_machine :state, :initial => :unmonitored do
+    state :unmonitored, :up, :down
+    state :starting, :stopping, :restarting
+
+    event :starting do
+      transition [:unmonitored, :down] => :starting
+    end
+
+    event :already_running do
+      transition [:unmonitored, :down, :up] => :up
+    end
+
+    event :started do
+      transition :starting => :up
+    end
+
+    event :crashed do
+      transition [:starting, :restarting, :up] => :down
+    end
+
+    event :stopping do
+      transition [:up, :restarting] => :stopping
+    end
+
+    event :stopped do
+      transition :stopping => :down
+    end
+
+    event :cant_kill do
+      transition :stopping => :up
+    end
+
+    event :restarting do
+      transition [:unmonitored, :up, :down] => :restarting
+    end
+
+    event :restarted do
+      transition :restarting => :up
+    end
+
+    event :unmonitoring do
+      transition any => :unmonitored
+    end
+
+    after_transition any => any, :do => :log_transition
+    after_transition any => any, :do => :check_triggers
+
+    after_transition any => :unmonitored, :do => :on_unmonitored
+
+    after_transition any-:up => :up, :do => :add_watchers
+    after_transition :up => any-:up, :do => :remove_watchers
+
+    after_transition any-:up => :up, :do => :add_children
+    after_transition any => [:unmonitored, :down], :do => :remove_children
+
+    after_transition :on => :crashed, :do => :on_crashed
+  end
+
+  def on_crashed
+    schedule :check_crash, Eye::Reason.new(:crashed)
+  end
+
+  def on_unmonitored
+    self.pid = nil
+  end
+
+  def log_transition(transition)
+    if transition.to_name != transition.from_name || @state_reason.is_a?(Eye::Reason::User)
+      @states_history.push transition.to_name, @state_reason
+      info "switch :#{transition.event} [:#{transition.from_name} => :#{transition.to_name}] #{@state_reason ? "(reason: #{@state_reason})" : nil}"
+    end
+  end
+
+end

data/lib/eye/process/states_history.rb

@@ -0,0 +1,66 @@
+class Eye::Process::StatesHistory < Eye::Utils::Tail
+
+  def push(state, reason = nil, tm = Time.now)
+    super(state: state, at: tm.to_i, reason: reason)
+  end
+
+  def states
+    self.map{|c| c[:state] }
+  end
+
+  def states_for_period(period, from_time = nil)
+    tm = Time.now - period
+    tm = [tm, from_time].max if from_time
+    tm = tm.to_f
+    self.select{|s| s[:at] >= tm }.map{|c| c[:state] }
+  end
+
+  def last_state
+    last[:state]
+  end
+
+  def last_reason
+    last[:reason] rescue nil
+  end
+
+  def last_state_changed_at
+    Time.at(last[:at])
+  end
+
+  def seq?(*seq)
+    str = states * ','
+    substr = seq.flatten * ','
+    str.include?(substr)
+  end
+
+  def end?(*seq)
+    str = states * ','
+    substr = seq.flatten * ','
+    str.end_with?(substr)
+  end
+
+  def any?(*seq)
+    states.any? do |st|
+      seq.flatten.include?(st)
+    end
+  end
+
+  def noone?(*seq)
+    !states.all? do |st|
+      seq.flatten.include?(st)
+    end
+  end
+
+  def all?(*seq)
+    states.all? do |st|
+      seq.flatten.include?(st)
+    end
+  end
+
+  def state_count(state)
+    states.count do |st|
+      st == state
+    end
+  end
+
+end

data/lib/eye/process/system.rb

@@ -0,0 +1,97 @@
+require 'timeout'
+
+module Eye::Process::System
+
+  def load_pid_from_file
+    if File.exists?(self[:pid_file_ex])
+      _pid = File.read(self[:pid_file_ex]).to_i
+      _pid > 0 ? _pid : nil
+    end
+  end
+
+  def set_pid_from_file
+    self.pid = load_pid_from_file
+  end
+
+  def save_pid_to_file
+    if self.pid
+      File.open(self[:pid_file_ex], 'w') do |f|
+        f.write self.pid
+      end
+      true
+    else
+      false
+    end
+  end
+
+  def clear_pid_file
+    info "delete pid_file: #{self[:pid_file_ex]}"
+    File.unlink(self[:pid_file_ex])
+    true
+  rescue
+    nil
+  end
+
+  def pid_file_ctime
+    File.ctime(self[:pid_file_ex]) rescue Time.now
+  end
+
+  def process_really_running?
+    process_pid_running?(self.pid)
+  end
+
+  def process_pid_running?(pid)
+    res = Eye::System.check_pid_alive(pid)
+    debug "process_really_running?: (#{pid}) #{res.inspect}"
+    !!res[:result]
+  end
+
+  def send_signal(code)
+    res = Eye::System.send_signal(self.pid, code)
+
+    msg = "send_signal #{code} to #{self.pid}"
+    msg += ", error<#{res[:error]}>" if res[:error]
+    info msg
+
+    res[:result] == :ok
+  end
+
+  def wait_for_condition(timeout, step = 0.1, &block)
+    res = nil
+    sumtime = 0
+
+    loop do
+      tm = Time.now
+      res = yield # note that yield can block actor here and timeout can be overhead
+      return res if res
+      sleep step.to_f
+      sumtime += (Time.now - tm)
+      return false if sumtime > timeout
+    end
+  end
+
+  def execute(cmd, cfg = {})
+    defer{ Eye::System::execute cmd, cfg }
+  end
+
+  def failsafe_load_pid
+    pid = load_pid_from_file
+
+    if !pid
+      # this is can be symlink changed case
+      sleep 0.1
+      pid = load_pid_from_file
+    end
+
+    pid
+  end
+
+  def failsafe_save_pid
+    save_pid_to_file
+    true
+  rescue => ex
+    log_ex(ex)
+    false
+  end
+
+end