reel-eye 0.3.1

data/lib/eye/process/states_history.rb

@@ -0,0 +1,66 @@
+class Eye::Process::StatesHistory < Eye::Utils::Tail
+
+  def push(state, reason = nil, tm = Time.now)
+    super(state: state, at: tm.to_i, reason: reason)
+  end
+
+  def states
+    self.map{|c| c[:state] }
+  end
+
+  def states_for_period(period, from_time = nil)
+    tm = Time.now - period
+    tm = [tm, from_time].max if from_time
+    tm = tm.to_f
+    self.select{|s| s[:at] >= tm }.map{|c| c[:state] }
+  end
+
+  def last_state
+    last[:state]
+  end
+
+  def last_reason
+    last[:reason]
+  end
+
+  def last_state_changed_at
+    Time.at(last[:at])
+  end
+
+  def seq?(*seq)
+    str = states * ','
+    substr = seq.flatten * ','
+    str.include?(substr)
+  end
+
+  def end?(*seq)
+    str = states * ','
+    substr = seq.flatten * ','
+    str.end_with?(substr)
+  end
+
+  def any?(*seq)
+    states.any? do |st|      
+      seq.flatten.include?(st)
+    end
+  end
+
+  def noone?(*seq)
+    !states.all? do |st|      
+      seq.flatten.include?(st)
+    end
+  end
+
+  def all?(*seq)
+    states.all? do |st|      
+      seq.flatten.include?(st)
+    end
+  end
+
+  def state_count(state)
+    states.count do |st|
+      st == state
+    end
+  end
+
+end

data/lib/eye/process/system.rb

@@ -0,0 +1,97 @@
+require 'timeout'
+
+module Eye::Process::System
+
+  def load_pid_from_file    
+    if File.exists?(self[:pid_file_ex])
+      _pid = File.read(self[:pid_file_ex]).to_i 
+      _pid > 0 ? _pid : nil
+    end
+  end
+
+  def set_pid_from_file
+    self.pid = load_pid_from_file
+  end
+
+  def save_pid_to_file
+    if self.pid
+      File.open(self[:pid_file_ex], 'w') do |f|
+        f.write self.pid
+      end
+      true
+    else
+      false
+    end
+  end
+
+  def clear_pid_file
+    File.unlink(self[:pid_file_ex])
+    true
+  rescue 
+    nil
+  end
+
+  def pid_file_ctime
+    File.ctime(self[:pid_file_ex]) rescue Time.now
+  end
+
+  def process_realy_running?
+    res = Eye::System.check_pid_alive(self.pid)
+    debug "process_realy_running?: (#{self.pid}) #{res.inspect}"
+    !!res[:result]
+  end
+
+  def send_signal(code)
+    res = Eye::System.send_signal(self.pid, code)
+
+    msg = "send_signal #{code} to #{self.pid}"
+    msg += ", error<#{res[:error]}>" if res[:error]
+    info msg
+
+    res[:result] == :ok
+  end
+
+  # non blocking actor timeout
+  def wait_for_condition(timeout, step = 0.1, &block)
+    defer{ wait_for_condition_sync(timeout, step, &block) }
+  end
+
+  def execute(cmd, cfg = {})
+    defer{ Eye::System::execute cmd, cfg }
+  end
+
+  def failsafe_load_pid
+    pid = load_pid_from_file
+
+    if !pid 
+      # this is can be symlink changed case
+      sleep 0.1
+      pid = load_pid_from_file
+    end
+
+    pid
+  end
+
+  def failsafe_save_pid
+    save_pid_to_file 
+    true
+  rescue => ex
+    error "failsafe_save_pid: #{ex.message}"
+    false
+  end
+
+private
+
+  def wait_for_condition_sync(timeout, step, &block)
+    res = nil
+
+    Timeout::timeout(timeout.to_f) do
+      sleep step.to_f until res = yield
+    end
+
+    res
+  rescue Timeout::Error
+    false
+  end
+
+end

data/lib/eye/process/trigger.rb

@@ -0,0 +1,54 @@
+module Eye::Process::Trigger
+
+  def add_triggers
+    if self[:triggers]
+      self[:triggers].each do |type, cfg|
+        add_trigger(cfg)
+      end      
+    end
+  end
+
+  def remove_triggers
+    self.triggers = []
+  end
+
+  def check_triggers
+    return if unmonitored?
+
+    self.triggers.each do |trigger|
+      unless trigger.check(self.states_history)
+        on_flapping(trigger) if trigger.class == Eye::Trigger::Flapping
+      end
+    end
+  end
+
+private
+
+  def add_trigger(cfg = {})
+    trigger = Eye::Trigger.create(cfg, logger.prefix)
+    self.triggers << trigger
+  end
+
+  def on_flapping(trigger)
+    notify :error, 'flapping!'
+    schedule :unmonitor, Eye::Reason.new(:flapping)
+
+    @retry_times ||= 0
+    retry_in = trigger.retry_in
+
+    return unless retry_in
+    return if trigger.retry_times && @retry_times >= trigger.retry_times
+
+    schedule_in(retry_in.to_f, :retry_action)
+  end
+
+  def retry_action
+    debug "trigger retry timer"
+    return unless unmonitored?
+    return unless state_reason.to_s.include?('flapping') # TODO: remove hackety
+
+    schedule :start, Eye::Reason.new(:'retry start after flapping')
+    @retry_times += 1
+  end
+
+end

data/lib/eye/process/validate.rb

@@ -0,0 +1,23 @@
+require 'shellwords'
+
+module Eye::Process::Validate
+
+  class Error < Exception; end
+
+  def validate(config)
+    if (str = config[:start_command])
+      # it should parse with Shellwords and not raise
+      spl = Shellwords.shellwords(str) * '#'
+
+      if config[:daemonize]
+        if spl =~ %r[sh#\-c|#&&#|;#]
+          raise Error, "#{config[:name]}, start_command in daemonize not supported shell concats like '&&'"
+        end
+      end
+    end
+
+    Shellwords.shellwords(config[:stop_command]) if config[:stop_command]
+    Shellwords.shellwords(config[:restart_command]) if config[:restart_command]
+  end
+
+end

data/lib/eye/process/watchers.rb

@@ -0,0 +1,69 @@
+module Eye::Process::Watchers
+
+  def add_watchers(force = false)
+    return unless self.up?
+
+    remove_watchers if force
+
+    if @watchers.blank?
+      # default watcher :check_alive
+      add_watcher(:check_alive, self[:check_alive_period]) do 
+        check_alive
+      end
+
+      # monitor childs pids
+      if self[:monitor_children]
+        add_watcher(:check_childs, self[:childs_update_period]) do 
+          add_or_update_childs
+        end
+      end
+
+      # monitor conditional watchers
+      start_checkers
+    else
+      warn 'try add_watchers, but its already here'
+    end
+  end
+  
+  def remove_watchers
+    @watchers.each{|_, h| h[:timer].cancel }
+    @watchers = {}
+  end
+
+private  
+
+  def add_watcher(type, period = 2, subject = nil, &block)
+    return if @watchers[type]
+
+    debug "add watcher #{type}(#{period})"
+
+    timer = every(period.to_f) do
+      debug "check #{type}"
+      block.call(subject)
+    end    
+
+    @watchers[type] ||= {:timer => timer, :subject => subject}
+  end  
+  
+  def start_checkers
+    self[:checks].each{|type, cfg| start_checker(cfg) }
+  end
+
+  def start_checker(cfg)
+    subject = Eye::Checker.create(pid, cfg, logger.prefix)
+
+    # ex: {:type => :memory, :every => 5.seconds, :below => 100.megabytes, :times => [3,5]}
+    add_watcher("check_#{cfg[:type]}".to_sym, subject.every, subject, &method(:watcher_tick).to_proc)
+  end
+
+  def watcher_tick(subject)
+    unless subject.check
+      return unless up?
+      
+      action = subject.fire || :restart
+      notify :warn, "Bounded #{subject.check_name}: #{subject.last_human_values} send to :#{action}"
+      schedule action, Eye::Reason.new("bounded #{subject.check_name}")
+    end
+  end
+
+end

data/lib/eye/reason.rb

@@ -0,0 +1,20 @@
+class Eye::Reason
+
+  def initialize(mes = nil)
+    @message = mes
+  end
+
+  def to_s
+    @message.to_s
+  end
+
+  def user?
+    self.class == User
+  end
+
+  class User < Eye::Reason
+    def to_s
+      "#{super} by user"
+    end
+  end
+end

data/lib/eye/server.rb

@@ -0,0 +1,52 @@
+require 'celluloid/io'
+require 'celluloid/autostart'
+
+class Eye::Server
+  include Celluloid::IO
+  
+  attr_reader :socket_path, :server
+  
+  def initialize(socket_path)
+    @socket_path = socket_path
+    @server = begin
+      UNIXServer.open(socket_path)
+    rescue Errno::EADDRINUSE
+      unlink_socket_file
+      UNIXServer.open(socket_path)
+    end
+  end
+
+  def run
+    loop { async.handle_connection @server.accept }
+  end
+
+  def handle_connection(socket)
+    command, *args = socket.readline.strip.split('|')
+    response = command(command, *args)
+    socket.write(Marshal.dump(response))
+    
+  rescue Errno::EPIPE
+    # client timeouted
+    # do nothing
+    
+  ensure
+    socket.close
+  end
+
+  def command(cmd, *args)
+    Eye::Control.command(cmd, *args)
+  end
+
+  def unlink_socket_file
+    File.delete(@socket_path) if @socket_path
+  rescue
+  end
+
+  finalizer :close_socket
+  
+  def close_socket
+    @server.close if @server
+    unlink_socket_file
+  end
+
+end

data/lib/eye/settings.rb

@@ -0,0 +1,46 @@
+require 'fileutils'
+
+module Eye::Settings
+  module_function
+  
+  def dir
+    if Process::UID.eid == 0 # root
+      '/var/run/eye'
+    else
+      File.expand_path(File.join(home, '.eye'))
+    end
+  end
+
+  def eyeconfig
+    if Process::UID.eid == 0 # root
+      '/etc/eye.conf'
+    else
+      File.expand_path(File.join(home, '.eyeconfig'))
+    end
+  end
+
+  def home
+    ENV['EYE_HOME'] || ENV['HOME']
+  end
+  
+  def path(path)
+    File.join(dir, path)
+  end
+
+  def ensure_eye_dir
+    FileUtils.mkdir_p( dir )
+  end
+
+  def socket_path
+    path('sock')
+  end
+  
+  def pid_path
+    path('pid')
+  end
+  
+  def client_timeout
+    5
+  end
+
+end

data/lib/eye/system.rb

@@ -0,0 +1,154 @@
+require 'shellwords'
+require 'pathname'
+
+module Eye::System
+  class << self
+    # Check that pid realy exits
+    # very fast
+    # return result hash
+    def check_pid_alive(pid)
+      res = if pid
+        ::Process.kill(0, pid)
+      else
+        false
+      end
+
+      {:result => res}
+    rescue => ex
+      {:error => ex}
+    end
+
+    # Check that pid realy exits
+    # very fast
+    # return true/false
+    def pid_alive?(pid)
+      res = check_pid_alive(pid)
+      !!res[:result]
+    end
+
+    # Send signal to process (uses for kill)
+    # code: TERM(15), KILL(9), QUIT(3), ...
+    def send_signal(pid, code = :TERM)      
+      code = 0 if code == '0'
+      if code.to_s.to_i != 0
+        code = code.to_i
+        code = -code if code < 0
+      end
+      code = code.to_s.upcase if code.is_a?(String) || code.is_a?(Symbol)
+
+      if pid
+        ::Process.kill(code, pid) 
+        {:result => :ok}
+      else
+        {:error => Exception.new('no_pid')}
+      end
+
+    rescue => ex
+      {:error => ex}
+    end
+
+    # Daemonize cmd, and detach
+    # options:
+    #   :pid_file
+    #   :working_dir
+    #   :environment
+    #   :stdin, :stdout, :stderr
+    def daemonize(cmd, cfg = {})
+      opts = spawn_options(cfg)
+      pid  = Process::spawn(prepare_env(cfg), *Shellwords.shellwords(cmd), opts)
+      Process.detach(pid)
+      {:pid => pid}
+      
+    rescue Errno::ENOENT, Errno::EACCES => ex
+      {:error => ex}
+    end
+
+    # Execute cmd with blocking, return status (be careful: inside actor blocks it mailbox, use with defer)
+    # options
+    #   :working_dir
+    #   :environment
+    #   :stdin, :stdout, :stderr
+    def execute(cmd, cfg = {})
+      opts = spawn_options(cfg)
+      pid  = Process::spawn(prepare_env(cfg), *Shellwords.shellwords(cmd), opts)
+
+      timeout = cfg[:timeout] || 1.second
+      Timeout.timeout(timeout) do
+        Process.waitpid(pid)
+      end
+
+      {:pid => pid}
+
+    rescue Timeout::Error => ex
+      if pid
+        Eye.warn "[#{cfg[:name]}] send signal 9 to #{pid} (because of timeouted<#{timeout}> execution)"
+        send_signal(pid, 9)
+      end
+      {:error => ex}
+
+    rescue Errno::ENOENT, Errno::EACCES => ex
+      {:error => ex}
+
+    ensure 
+      Process.detach(pid) if pid
+    end
+
+    # get table
+    # {pid => {:rss =>, :cpu =>, :ppid => , :start_time => }}
+    # slow
+    def ps_aux
+      str = Process.send('`', PS_AUX_CMD).force_encoding('binary')
+      h = {}
+      str.each_line do |line|
+        chunk = line.strip.split(/\s+/)
+        h[chunk[0].to_i] = { :ppid => chunk[1].to_i, :cpu => chunk[2].to_i, 
+          :rss => chunk[3].to_i, :start_time => chunk[4] }
+      end
+      h
+    end
+
+    # normalize file
+    def normalized_file(file, working_dir = nil)
+      Pathname.new(file).expand_path(working_dir).to_s
+    end
+
+    def host
+      @host ||= `hostname`.chomp
+    end
+
+    # set host for tests
+    def host=(hostname)
+      @host = hostname
+    end
+
+  private
+
+    PS_AUX_CMD = if RUBY_PLATFORM.include?('darwin')
+      'ps axo pid,ppid,pcpu,rss,start'
+    else
+      'ps axo pid,ppid,pcpu,rss,start_time'
+    end
+
+    def spawn_options(config = {})
+      o = {pgroup: true, chdir: config[:working_dir] || '/'}
+      o.update(out: [config[:stdout], 'a']) if config[:stdout]
+      o.update(err: [config[:stderr], 'a']) if config[:stderr]
+      o.update(in: config[:stdin]) if config[:stdin]
+      o
+    end
+
+    def prepare_env(config = {})
+      env = {}
+
+      (config[:environment] || {}).each do |k,v|
+        env[k.to_s] = v.to_s if v
+      end
+
+      # set PWD for unicorn respawn
+      env['PWD'] = config[:working_dir] if config[:working_dir]
+
+      env
+    end
+  end
+
+end