eye 0.1.11

data/lib/eye/process/config.rb

@@ -0,0 +1,70 @@
+module Eye::Process::Config
+
+  DEFAULTS = {
+    :keep_alive => true, # restart when crushed
+    :check_alive_period => 5.seconds,
+
+    :start_timeout => 15.seconds,
+    :stop_timeout => 10.seconds,
+    :restart_timeout => 10.seconds,
+
+    :start_grace => 2.5.seconds, 
+    :stop_grace => 0.5.seconds, 
+    :restart_grace => 0.5.seconds, 
+
+    :daemonize => false,
+    :auto_start => true, # auto start on monitor action
+
+    :childs_update_period => 30.seconds
+  }
+
+  def prepare_config(new_config)
+    h = DEFAULTS.merge(new_config)
+    h[:pid_file_ex] = Eye::System.normalized_file(h[:pid_file], h[:working_dir]) if h[:pid_file]
+    h[:checks] = {} if h[:checks].blank?
+    h[:triggers] = {} if h[:triggers].blank?
+    h[:childs_update_period] = h[:monitor_children][:childs_update_period] if h[:monitor_children] && h[:monitor_children][:childs_update_period]
+
+    # check speedy flapping by default
+    if h[:triggers].blank? || !h[:triggers][:flapping]
+      h[:triggers] ||= {}
+      h[:triggers][:flapping] = {:type => :flapping, :times => 10, :within => 10.seconds}
+    end
+    
+    h[:stdout] = Eye::System.normalized_file(h[:stdout], h[:working_dir]) if h[:stdout]
+    h[:stderr] = Eye::System.normalized_file(h[:stderr], h[:working_dir]) if h[:stderr]
+
+    h    
+  end
+
+  def c(name)
+    @config[name]
+  end
+  
+  def [](name)
+    @config[name]
+  end
+  
+  def update_config(new_config = {})
+    new_config = prepare_config(new_config)
+    @config = new_config
+    @full_name = nil
+
+    debug "update config to: #{@config.inspect}"
+
+    remove_triggers
+    add_triggers
+
+    if up?
+      # rebuild checks for this process
+      from_up; on_up
+    end    
+  end
+
+  # is pid_file under Eye::Process control, or not
+  def control_pid?
+    return self[:control_pid] unless self[:control_pid].nil?
+    !!self[:daemonize]
+  end
+  
+end

data/lib/eye/process/controller.rb

@@ -0,0 +1,72 @@
+module Eye::Process::Controller
+
+  def send_command(command, *args)
+    schedule command, *args, "#{command} by user"
+  end
+
+  def start
+    res = if set_pid_from_file
+      if process_realy_running?
+        info "process from pid_file(#{self.pid}) found and already running, so :up"
+        switch :already_running
+        :ok
+      else
+        info "pid_file found, but process in pid_file(#{self.pid}) not found, starting..."
+        start_process
+      end
+    else
+      info 'pid_file not found, so starting process...'
+      start_process
+    end
+
+    res
+  end
+
+  def stop
+    stop_process
+    switch :unmonitoring
+  end
+
+  def restart
+    restart_process
+  end
+
+  def monitor
+    if self[:auto_start]
+      start
+    else
+      if try_update_pid_from_file
+        info "process from pid_file(#{self.pid}) found and already running, so :up"
+        switch :already_running
+      else
+        warn "process not found, so :unmonitor"
+        schedule :unmonitor, 'not found'
+      end
+    end
+  end
+
+  def unmonitor
+    switch :unmonitoring
+  end
+  
+  def delete
+    if self[:stop_on_delete]
+      info 'process has stop_on_delete option, so sync-stop it first'
+      stop
+    end
+
+    remove_watchers
+    remove_childs
+    remove_triggers
+
+    terminate
+  end
+
+  def signal(sig = 0)
+    if self.pid
+      res = send_signal(sig) 
+      info "send signal #{sig} to #{self.pid} = #{res}"
+    end
+  end
+  
+end

data/lib/eye/process/data.rb

@@ -0,0 +1,46 @@
+module Eye::Process::Data
+
+  # logger tag
+  def full_name
+    @full_name ||= [self[:application], (self[:group] == '__default__') ? nil : self[:group], self[:name]].compact.join(':')
+  end
+
+  def status_data(debug = false)
+    p_st = self_status_data(debug)
+
+    if childs.present?
+      p_st.merge(:subtree => Eye::Utils::AliveArray.new(childs.values).map{|c| c.status_data(debug) } )
+    elsif self[:monitor_children] && self.up?
+      p_st.merge(:subtree => [{name: '=loading childs='}])
+    else
+      # common state
+      p_st
+    end
+  end
+
+  def self_status_data(debug = false)
+    h = { name: name, state: state, 
+          type: (self.class == Eye::ChildProcess ? :child_process : :process), 
+          resources: Eye::SystemResources.resources(pid) }
+
+    if @states_history
+      h.merge!( state_changed_at: @states_history.last[:at],
+                state_reason: @states_history.last[:reason] )
+    end
+
+    h.merge!(debug: debug_data) if debug
+    h.merge!(current_command: current_scheduled_command) if current_scheduled_command
+
+    h    
+  end
+
+  def debug_data
+    { queue: scheduler_actions_list, watchers: @watchers.keys }
+  end
+
+  def sub_object?(obj)
+    # we not recognize childs
+    false
+  end
+
+end

data/lib/eye/process/monitor.rb

@@ -0,0 +1,97 @@
+module Eye::Process::Monitor
+
+private
+
+  def check_alive_with_refresh_pid_if_needed
+    if process_realy_running?
+      return true
+
+    else
+      warn 'process not realy running'
+      try_update_pid_from_file
+    end
+  end
+
+  def try_update_pid_from_file
+    # if pid file was rewrited
+    newpid = load_pid_from_file
+    if newpid != self.pid 
+      info "process changed pid to #{newpid}, updating..." if self.pid
+      self.pid = newpid
+
+      if process_realy_running?
+        return true
+      else
+        warn "process with new_pid #{newpid} not found"
+        return false          
+      end
+    else
+      debug 'process not found'
+      return false
+    end
+  end
+
+  REWRITE_FACKUP_PIDFILE_PERIOD = 2.minutes
+  
+  def check_alive
+    if up?
+
+      # check that process runned
+      unless process_realy_running?
+        warn 'check_alive: process not found'
+        notify :warn, 'crushed!'
+        switch :crushed
+      else
+        # check that pid_file still here
+        ppid = failsafe_load_pid
+        
+        if ppid != self.pid
+          msg = "check_alive: pid_file(#{self[:pid_file]}) changes by itself (#{self.pid}) => (#{ppid})"
+          if control_pid?
+            msg += ", not correct, pid_file is under eye control, so rewrited back #{self.pid}"
+            save_pid_to_file rescue msg += ', (Can`t rewrite pid_file O_o)'
+          else
+            if ppid == nil
+              msg += ', rewrited because empty'
+              save_pid_to_file rescue msg += ', (Can`t rewrite pid_file O_o)'
+            elsif (Time.now - pid_file_ctime > REWRITE_FACKUP_PIDFILE_PERIOD)
+              msg += ", > #{REWRITE_FACKUP_PIDFILE_PERIOD.inspect} ago, so rewrited (even if pid_file not under eye control)"
+              save_pid_to_file rescue msg += ', (Can`t rewrite pid_file O_o)'
+            else
+              msg += ', not under eye control, so ignored'
+            end
+          end
+
+          warn msg
+        end
+      end
+    end
+  end
+
+  def failsafe_load_pid
+    pid = load_pid_from_file
+
+    if !pid 
+      # this is can be symlink changed case
+      sleep 0.1
+      pid = load_pid_from_file
+    end
+
+    pid
+  end
+
+  def check_crush
+    if down?
+      if self[:keep_alive] && !@flapping
+        warn 'check crushed: process is down'
+        schedule :start, 'crushed'
+      else
+        warn 'check crushed: process is down, and flapping happens (or not keep_alive option)'
+        schedule :unmonitor, 'flapping'
+      end
+    end
+  end
+
+  public :check_crush # bug of celluloid 0.12
+
+end

data/lib/eye/process/notify.rb

@@ -0,0 +1,17 @@
+module Eye::Process::Notify
+
+  # notify to user:
+  # 1) process crushed by itself, and we restart it
+  # 2) checker fire to restart process +
+  # 3) flapping + switch to unmonitored
+
+  # level = [:warn, :crit]
+
+  # TODO: add mail, jabber here
+  def notify(level, msg)
+    if level != :warn
+      warn "!!!!!!!! NOTIFY: #{level}, #{msg} !!!!!!!!!!!"
+    end
+  end
+
+end

data/lib/eye/process/scheduler.rb

@@ -0,0 +1,50 @@
+module Eye::Process::Scheduler
+
+  # ex: schedule :update_config, config, "reason: update_config"
+  def schedule(command, *args, &block)
+    if scheduler.alive?
+      reason = if args.present? && [String, Symbol].include?(args[-1].class)
+        args.pop
+      end
+
+      info "schedule :#{command} (#{reason})"
+      scheduler.add_wo_dups(:scheduled_action, command, {:args => args, :reason => reason}, &block)
+    end
+  end
+
+  def scheduled_action(command, h = {}, &block)
+    reason = h.delete(:reason)
+    info "=> #{command} #{h[:args].present? ? "#{h[:args]*',' }" : nil}(#{reason})"
+
+    @current_scheduled_command = command
+    @last_scheduled_command = command
+    @last_scheduled_reason = reason
+    @last_scheduled_at = Time.now
+
+    send(command, *h[:args], &block)
+    @current_scheduled_command = nil
+    info "<= #{command}"
+  end
+
+  def scheduler_actions_list
+    scheduler.list.map{|c| c[:args].first rescue nil }.compact
+  end
+
+  def finalize
+    remove_scheduler
+  end
+
+  attr_accessor :current_scheduled_command
+  attr_accessor :last_scheduled_command, :last_scheduled_reason, :last_scheduled_at
+
+private
+
+  def remove_scheduler
+    @scheduler.terminate if @scheduler && @scheduler.alive?
+  end
+
+  def scheduler
+    @scheduler ||= Eye::Utils::CelluloidChain.new(current_actor)
+  end
+
+end

data/lib/eye/process/states.rb

@@ -0,0 +1,92 @@
+gem 'state_machine'
+require 'state_machine'
+
+class Eye::Process
+
+  # do transition
+  def switch(name, reason = nil)
+    @state_reason = reason || last_scheduled_reason
+    self.send("#{name}!")
+  end
+
+  state_machine :state, :initial => :unmonitored do
+    state :unmonitored, :up, :down
+    state :starting, :stopping, :restarting
+
+    event :starting do
+      transition [:unmonitored, :down] => :starting
+    end
+
+    event :already_running do
+      transition [:unmonitored, :down, :up] => :up
+    end
+
+    event :started do
+      transition :starting => :up
+    end
+
+    event :crushed do
+      transition [:starting, :restarting, :up] => :down
+    end
+
+    event :stopping do
+      transition [:up, :restarting] => :stopping
+    end
+
+    event :stopped do
+      transition :stopping => :down
+    end
+
+    event :cant_kill do
+      transition :stopping => :up
+    end
+
+    event :restarting do
+      transition [:unmonitored, :up, :down] => :restarting
+    end
+
+    event :restarted do
+      transition :restarting => :up
+    end
+
+    event :unmonitoring do
+      transition any => :unmonitored
+    end
+
+    after_transition :on => :crushed, :do => :on_crushed
+    after_transition any => :unmonitored, :do => :on_unmonitored
+    after_transition any-:up => :up, :do => :on_up
+    after_transition :up => any-:up, :do => :from_up
+    after_transition any => any, :do => :log_transition
+    after_transition any => any, :do => :upd_for_triggers
+  end
+
+  def on_crushed
+    schedule :check_crush, 'crushed'
+  end
+
+  def on_unmonitored
+    self.flapping = false
+    self.pid = nil
+  end
+
+  def on_up
+    add_watchers
+    add_childs
+  end
+
+  def from_up
+    remove_watchers
+    remove_childs
+  end
+
+  def log_transition(transition)
+    @states_history.push transition.to_name, @state_reason
+    info "switch :#{transition.event} [:#{transition.from_name} => :#{transition.to_name}] #{@state_reason ? "(#{@state_reason})" : nil}"
+  end
+
+  def upd_for_triggers(transition)
+    check_triggers
+  end
+
+end