reel-eye 0.3.1

data/lib/eye/process/config.rb

@@ -0,0 +1,70 @@
+module Eye::Process::Config
+
+  DEFAULTS = {
+    :keep_alive => true, # restart when crashed
+    :check_alive_period => 5.seconds,
+
+    :start_timeout => 15.seconds,
+    :stop_timeout => 10.seconds,
+    :restart_timeout => 10.seconds,
+
+    :start_grace => 2.5.seconds, 
+    :stop_grace => 0.5.seconds, 
+    :restart_grace => 0.5.seconds, 
+
+    :daemonize => false,
+    :auto_start => true, # auto start on monitor action
+
+    :childs_update_period => 30.seconds
+  }
+
+  def prepare_config(new_config)
+    h = DEFAULTS.merge(new_config)
+    h[:pid_file_ex] = Eye::System.normalized_file(h[:pid_file], h[:working_dir]) if h[:pid_file]
+    h[:checks] = {} if h[:checks].blank?
+    h[:triggers] = {} if h[:triggers].blank?
+    h[:childs_update_period] = h[:monitor_children][:childs_update_period] if h[:monitor_children] && h[:monitor_children][:childs_update_period]
+
+    # check speedy flapping by default
+    if h[:triggers].blank? || !h[:triggers][:flapping]
+      h[:triggers] ||= {}
+      h[:triggers][:flapping] = {:type => :flapping, :times => 10, :within => 10.seconds}
+    end
+    
+    h[:stdout] = Eye::System.normalized_file(h[:stdout], h[:working_dir]) if h[:stdout]
+    h[:stderr] = Eye::System.normalized_file(h[:stderr], h[:working_dir]) if h[:stderr]
+
+    h    
+  end
+
+  def c(name)
+    @config[name]
+  end
+  
+  def [](name)
+    @config[name]
+  end
+  
+  def update_config(new_config = {})
+    new_config = prepare_config(new_config)
+    @config = new_config
+    @full_name = nil
+
+    debug "update config to: #{@config.inspect}"
+
+    remove_triggers
+    add_triggers
+
+    if up?
+      # rebuild checks for this process
+      from_up; on_up
+    end    
+  end
+
+  # is pid_file under Eye::Process control, or not
+  def control_pid?
+    return self[:control_pid] unless self[:control_pid].nil?
+    !!self[:daemonize]
+  end
+  
+end

data/lib/eye/process/controller.rb

@@ -0,0 +1,76 @@
+module Eye::Process::Controller
+
+  def send_command(command, *args)
+    schedule command, *args, Eye::Reason::User.new(command)
+  end
+
+  def start
+    res = if set_pid_from_file
+      if process_realy_running?
+        info "process from pid_file(#{self.pid}) found and already running, so :up"
+        switch :already_running
+        :ok
+      else
+        info "pid_file found, but process in pid_file(#{self.pid}) not found, starting..."
+        start_process
+      end
+    else
+      info 'pid_file not found, so starting process...'
+      start_process
+    end
+
+    res
+  end
+
+  def stop
+    stop_process
+    switch :unmonitoring
+  end
+
+  def restart
+    unless pid # unmonitored case
+      try_update_pid_from_file
+    end
+
+    restart_process
+  end
+
+  def monitor
+    if self[:auto_start]
+      start
+    else
+      if try_update_pid_from_file
+        info "process from pid_file(#{self.pid}) found and already running, so :up"
+        switch :already_running
+      else
+        warn "process not found, so :unmonitor"
+        schedule :unmonitor, Eye::Reason.new(:'not found')
+      end
+    end
+  end
+
+  def unmonitor
+    switch :unmonitoring
+  end
+  
+  def delete
+    if self[:stop_on_delete]
+      info 'process has stop_on_delete option, so sync-stop it first'
+      stop
+    end
+
+    remove_watchers
+    remove_childs
+    remove_triggers
+
+    terminate
+  end
+
+  def signal(sig = 0)
+    if self.pid
+      res = send_signal(sig) 
+      info "send signal #{sig} to #{self.pid} = #{res}"
+    end
+  end
+  
+end

data/lib/eye/process/data.rb

@@ -0,0 +1,47 @@
+module Eye::Process::Data
+
+  # logger tag
+  def full_name
+    @full_name ||= [self[:application], (self[:group] == '__default__') ? nil : self[:group], self[:name]].compact.join(':')
+  end
+
+  def status_data(debug = false)
+    p_st = self_status_data(debug)
+
+    if childs.present?
+      p_st.merge(:subtree => Eye::Utils::AliveArray.new(childs.values).map{|c| c.status_data(debug) } )
+    elsif self[:monitor_children] && self.up?
+      p_st.merge(:subtree => [{name: '=loading childs='}])
+    else
+      # common state
+      p_st
+    end
+  end
+
+  def self_status_data(debug = false)
+    h = { name: name, state: state, 
+          type: (self.class == Eye::ChildProcess ? :child_process : :process), 
+          resources: Eye::SystemResources.resources(pid) }
+
+    if @states_history
+      h.merge!( state_changed_at: @states_history.last_state_changed_at,
+                state_reason: @states_history.last_reason )
+    end
+
+    h.merge!(debug: debug_data) if debug
+    h.merge!(current_command: current_scheduled_command) if current_scheduled_command
+
+    h    
+  end
+
+  def debug_data
+    { queue: scheduler_actions_list, watchers: @watchers.keys }
+  end
+
+  def sub_object?(obj)
+    return false if self.class == Eye::ChildProcess
+    self.childs.each { |_, child| return true if child == obj }
+    false
+  end
+
+end

data/lib/eye/process/monitor.rb

@@ -0,0 +1,95 @@
+module Eye::Process::Monitor
+
+private
+
+  def check_alive_with_refresh_pid_if_needed
+    if process_realy_running?
+      return true
+
+    else
+      warn 'process not realy running'
+      try_update_pid_from_file
+    end
+  end
+
+  def try_update_pid_from_file
+    # if pid file was rewrited
+    newpid = load_pid_from_file
+    if newpid != self.pid 
+      info "process changed pid to #{newpid}, updating..." if self.pid
+      self.pid = newpid
+
+      if process_realy_running?
+        return true
+      else
+        warn "process with new_pid #{newpid} not found"
+        return false          
+      end
+    else
+      debug 'process not found'
+      return false
+    end
+  end
+
+  REWRITE_FACKUP_PIDFILE_PERIOD = 2.minutes
+  
+  def check_alive
+    if up?
+
+      # check that process runned
+      unless process_realy_running?
+        warn "check_alive: process(#{self.pid}) not found!"
+        notify :info, 'crashed!'
+        switch :crashed, Eye::Reason.new(:crashed)
+      else
+        # check that pid_file still here
+        ppid = failsafe_load_pid
+        
+        if ppid != self.pid
+          msg = "check_alive: pid_file(#{self[:pid_file]}) changes by itself (pid:#{self.pid}) => (pid:#{ppid})"
+          if control_pid?
+            msg += ", not correct, pid_file is under eye control, so rewrited back pid:#{self.pid}"
+            unless failsafe_save_pid
+              msg += ', (Can`t rewrite pid_file O_o)'
+            end
+          else
+            if ppid == nil
+              msg += ', rewrited because empty'
+              unless failsafe_save_pid
+                msg += ', (Can`t rewrite pid_file O_o)'
+              end
+            elsif (Time.now - pid_file_ctime > REWRITE_FACKUP_PIDFILE_PERIOD)
+              msg += ", > #{REWRITE_FACKUP_PIDFILE_PERIOD.inspect} ago, so rewrited (even if pid_file not under eye control)"
+              unless failsafe_save_pid
+                msg += ', (Can`t rewrite pid_file O_o)'
+              end
+            else
+              msg += ', not under eye control, so ignored'
+            end
+          end
+
+          warn msg
+        end
+      end
+    end
+  end
+
+  def check_crash
+    if down?
+      if self[:keep_alive]
+        warn 'check crashed: process is down'
+        schedule :restore, Eye::Reason.new(:crashed)
+      else
+        warn 'check crashed: process without keep_alive'
+        schedule :unmonitor, Eye::Reason.new(:crashed)
+      end
+    else
+      debug 'check crashed: skipped, process is not in down'
+    end
+  end
+
+  def restore
+    start if down?
+  end
+
+end

data/lib/eye/process/notify.rb

@@ -0,0 +1,32 @@
+module Eye::Process::Notify
+
+  # notify to user:
+  # 1) process crashed by itself, and we restart it [:info]
+  # 2) checker bounded to restart process [:warn]
+  # 3) flapping + switch to unmonitored [:error]
+
+  LEVELS = {:debug => 0, :info => 1, :warn => 2, :error => 3, :fatal => 4}
+
+  def notify(level, msg)
+    # logging it
+    error "NOTIFY: #{msg}" if ilevel(level) > ilevel(:info)
+
+    # send notifies
+    if self[:notify].present?
+      message = {:message => msg, :name => name, 
+        :full_name => full_name, :pid => pid, :host => Eye::System.host, :level => level,
+        :at => Time.now }
+
+      self[:notify].each do |contact, not_level|
+        Eye::Notify.notify(contact, message) if ilevel(level) >= ilevel(not_level)
+      end
+    end
+  end
+
+private
+
+  def ilevel(level)
+    LEVELS[level].to_i
+  end
+
+end

data/lib/eye/process/scheduler.rb

@@ -0,0 +1,78 @@
+module Eye::Process::Scheduler
+
+  # ex: schedule :update_config, config, "reason: update_config"
+  def schedule(command, *args, &block)
+    if scheduler.alive?
+      unless self.respond_to?(command, true)
+        warn "object not support :#{command} to schedule"
+        return
+      end
+
+      reason = if args.present? && args[-1].kind_of?(Eye::Reason)
+        args.pop
+      end
+
+      info "schedule :#{command} #{reason ? "(reason: #{reason})" : nil}"
+
+      if reason.class == Eye::Reason
+        # for auto reasons
+        # skip already running commands and all in chain
+        scheduler.add_wo_dups_current(:scheduled_action, command, {:args => args, :reason => reason}, &block)  
+      else
+        # for manual, or without reason
+        # skip only for last in chain
+        scheduler.add_wo_dups(:scheduled_action, command, {:args => args, :reason => reason}, &block)
+      end
+    end
+  end
+
+  def schedule_in(interval, command, *args, &block)
+    debug "schedule_in #{interval} :#{command} #{args}"
+    after(interval.to_f) do
+      debug "scheduled_in #{interval} :#{command} #{args}"
+      schedule(command, *args, &block)
+    end
+  end
+
+  def scheduled_action(command, h = {}, &block)
+    reason = h.delete(:reason)
+    info "=> #{command} #{h[:args].present? ? "#{h[:args]*',' }" : nil} #{reason ? "(reason: #{reason})" : nil}"
+
+    @current_scheduled_command = command
+    @last_scheduled_command = command
+    @last_scheduled_reason = reason
+    @last_scheduled_at = Time.now
+
+    send(command, *h[:args], &block)
+    @current_scheduled_command = nil
+    info "<= #{command}"
+
+    schedule_history.push(command, reason, @last_scheduled_at.to_i)
+  end
+
+  def scheduler_actions_list
+    scheduler.list.map{|c| c[:args].first rescue nil }.compact
+  end
+  
+  def self.included(base)
+    base.finalizer :remove_scheduler
+  end
+
+  attr_accessor :current_scheduled_command
+  attr_accessor :last_scheduled_command, :last_scheduled_reason, :last_scheduled_at
+
+  def schedule_history
+    @schedule_history ||= Eye::Process::StatesHistory.new(50)
+  end
+
+private
+
+  def remove_scheduler
+    @scheduler.terminate if @scheduler && @scheduler.alive?
+  end
+  
+  def scheduler
+    @scheduler ||= Eye::Utils::CelluloidChain.new(current_actor)
+  end
+
+end

data/lib/eye/process/states.rb

@@ -0,0 +1,86 @@
+require 'state_machine'
+
+class Eye::Process
+
+  # do transition
+  def switch(name, reason = nil)
+    @state_reason = reason || last_scheduled_reason
+    self.send("#{name}!")
+  end
+
+  state_machine :state, :initial => :unmonitored do
+    state :unmonitored, :up, :down
+    state :starting, :stopping, :restarting
+
+    event :starting do
+      transition [:unmonitored, :down] => :starting
+    end
+
+    event :already_running do
+      transition [:unmonitored, :down, :up] => :up
+    end
+
+    event :started do
+      transition :starting => :up
+    end
+
+    event :crashed do
+      transition [:starting, :restarting, :up] => :down
+    end
+
+    event :stopping do
+      transition [:up, :restarting] => :stopping
+    end
+
+    event :stopped do
+      transition :stopping => :down
+    end
+
+    event :cant_kill do
+      transition :stopping => :up
+    end
+
+    event :restarting do
+      transition [:unmonitored, :up, :down] => :restarting
+    end
+
+    event :restarted do
+      transition :restarting => :up
+    end
+
+    event :unmonitoring do
+      transition any => :unmonitored
+    end
+
+    after_transition any => :unmonitored, :do => :on_unmonitored
+    after_transition any-:up => :up, :do => :on_up
+    after_transition :up => any-:up, :do => :from_up
+    after_transition any => any, :do => :log_transition
+    after_transition any => any, :do => :check_triggers
+    after_transition :on => :crashed, :do => :on_crashed
+  end
+
+  def on_crashed
+    schedule :check_crash, Eye::Reason.new(:crashed)
+  end
+
+  def on_unmonitored
+    self.pid = nil
+  end
+
+  def on_up
+    add_watchers
+    add_childs
+  end
+
+  def from_up
+    remove_watchers
+    remove_childs
+  end
+
+  def log_transition(transition)
+    @states_history.push transition.to_name, @state_reason
+    info "switch :#{transition.event} [:#{transition.from_name} => :#{transition.to_name}] #{@state_reason ? "(reason: #{@state_reason})" : nil}"
+  end
+
+end