reel-eye 0.3.1

data/examples/thin-farm.eye

@@ -0,0 +1,29 @@
+RUBY = 'ruby'
+BUNDLE = 'bundle'
+
+Eye.load("process_thin.rb")
+
+Eye.config do
+  logger "/tmp/eye.log"
+end
+
+Eye.app 'thin-farm' do
+  working_dir File.expand_path(File.join(File.dirname(__FILE__), %w[ processes ]))
+  env "RAILS_ENV" => "production"
+
+  stop_on_delete true # this option means, when we change pids and load config, 
+                      # deleted processes will be stops
+
+  triggers :flapping, :times => 10, :within => 1.minute
+  checks :memory, :below => 60.megabytes, :every => 30.seconds, :times => 5
+
+  group :web do
+    chain :action => :restart, :grace => 5.seconds
+    chain :action => :start, :grace => 0.2.seconds
+
+    (5555..5560).each do |port|
+      thin self, port
+    end
+  end
+
+end

data/examples/unicorn.eye

@@ -0,0 +1,31 @@
+# Example: now to run unicorn, and monitor its childs processes
+
+RUBY = '/usr/local/ruby/1.9.3/bin/ruby' # ruby on the server
+RAILS_ENV = 'production'
+
+Eye.application "rails_unicorn" do
+  env "RAILS_ENV" => RAILS_ENV, "PATH" => "#{File.dirname(RUBY)}:#{ENV['PATH']}"
+  working_dir "/projects/rails_unicorn"
+  
+  process("unicorn") do
+    pid_file "tmp/pids/unicorn.pid"
+    start_command "#{RUBY} ./bin/unicorn -Dc ./config/unicorn.rb -E #{RAILS_ENV}"
+    stop_command "kill -QUIT {{PID}}"
+    restart_command "kill -USR2 {{PID}}"
+    stdall "log/unicorn.log"
+
+    checks :cpu, :every => 30, :below => 80, :times => 3
+    checks :memory, :every => 30, :below => 150.megabytes, :times => [3,5]
+
+    start_timeout 30.seconds
+    stop_grace 5.seconds
+    restart_grace 30.seconds
+
+    monitor_children do
+      stop_command "kill -QUIT {{PID}}"
+      checks :cpu, :every => 30, :below => 80, :times => 3
+      checks :memory, :every => 30, :below => 150.megabytes, :times => [3,5]
+    end
+  end
+
+end

data/eye.gemspec

@@ -0,0 +1,42 @@
+require File.expand_path('../lib/eye', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = "Konstantin Makarchev"
+  gem.email         = "kostya27@gmail.com"
+
+  gem.description   = gem.summary = \
+    %q{Process monitoring tool. An alternative to God and Bluepill. With Bluepill like config syntax. Requires MRI Ruby >= 1.9.2. Uses Celluloid and Celluloid::IO.}
+  gem.homepage      = "http://github.com/kostya/eye"
+
+  gem.files         = `git ls-files`.split($\).reject{|n| n =~ %r[png|gif\z]}.reject{|n| n =~ %r[^(test|spec|features)/]}
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  #gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "reel-eye"
+  gem.require_paths = ["lib"]
+  gem.version       = Eye::VERSION
+  gem.license       = "MIT"
+
+  gem.required_ruby_version     = '>= 1.9.2' # because of celluloid
+  gem.required_rubygems_version = '>= 1.3.6'  
+  
+  gem.add_dependency 'celluloid',     '~> 0.13.0'
+  gem.add_dependency 'celluloid-io',  '~> 0.13.0'
+  gem.add_dependency 'state_machine', '< 1.2'
+  gem.add_dependency 'activesupport', '~> 3.2.0'
+  gem.add_dependency 'thor'
+
+  gem.add_dependency 'reel',          '~> 0.4.0.pre'
+  gem.add_dependency 'cuba'
+
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'rspec'
+  gem.add_development_dependency 'simplecov'
+  gem.add_development_dependency 'rr'
+  gem.add_development_dependency 'ruby-graphviz'
+  gem.add_development_dependency 'forking'
+  gem.add_development_dependency 'fakeweb'
+  gem.add_development_dependency 'eventmachine', ">= 1.0.3"
+  gem.add_development_dependency 'sinatra'
+  gem.add_development_dependency 'thin'
+  gem.add_development_dependency 'xmpp4r'
+end

data/lib/eye.rb

@@ -0,0 +1,28 @@
+module Eye  
+  VERSION   = "0.3.1"
+  ABOUT     = "ReelEye v#{VERSION} (c) 2012-2013 @kostya"
+  PROCLINE  = "reel-eye monitoring v#{VERSION}"
+
+  autoload :Process,        'eye/process'
+  autoload :ChildProcess,   'eye/child_process'
+  autoload :Server,         'eye/server'
+  autoload :Logger,         'eye/logger'
+  autoload :System,         'eye/system'
+  autoload :SystemResources,'eye/system_resources'  
+  autoload :Checker,        'eye/checker'
+  autoload :Trigger,        'eye/trigger'
+  autoload :Group,          'eye/group'
+  autoload :Dsl,            'eye/dsl'  
+  autoload :Application,    'eye/application'  
+  autoload :Settings,       'eye/settings'
+  autoload :Client,         'eye/client'
+  autoload :Utils,          'eye/utils'
+  autoload :Notify,         'eye/notify'
+  autoload :Config,         'eye/config'
+  autoload :Reason,         'eye/reason'
+  
+  autoload :Controller,     'eye/controller'
+  autoload :Control,        'eye/control'
+
+  autoload :Http,           'eye/http'
+end

data/lib/eye/application.rb

@@ -0,0 +1,74 @@
+class Eye::Application
+
+  attr_reader :groups, :name
+
+  include Eye::Logger::Helpers
+
+  def initialize(name, config = {})
+    @groups = Eye::Utils::AliveArray.new
+    @name = name
+    @logger = Eye::Logger.new(full_name)
+    @config = config
+    debug 'created'
+  end
+
+  def full_name
+    @name
+  end
+
+  def update_config(cfg)
+    @config = cfg
+  end
+
+  def add_group(group)
+    @groups << group
+  end
+
+  # sort processes in name order
+  def resort_groups
+    @groups = @groups.sort_by{|gr| gr.name == '__default__' ? 'zzzzz' : gr.name }
+  end
+
+  def status_data(debug = false)
+    h = { name: @name, type: :application, subtree: @groups.map{|gr| gr.status_data(debug) }}
+    h.merge!(debug: debug_data) if debug
+    h
+  end
+
+  def status_data_short
+    h = Hash.new 0
+    @groups.each do |c| 
+      c.processes.each do |p|
+        h[p.state] += 1
+      end
+    end
+    str = h.sort_by{|a,b| a}.map{|k, v| "#{k}:#{v}" } * ', '
+    { name: @name, type: :application, state: str}
+  end
+
+  def debug_data
+  end
+
+  def send_command(command, *args)
+    debug "send_command #{command} #{args}"
+    
+    @groups.each do |group|
+      group.send_command(command, *args)
+    end
+  end
+
+  def alive?
+    true # emulate celluloid actor method
+  end
+
+  def sub_object?(obj)
+    res = @groups.include?(obj)
+    res = @groups.any?{|gr| gr.sub_object?(obj)} if !res
+    res
+  end
+
+  def processes
+    Eye::Utils::AliveArray.new(@groups.map{|gr| gr.processes.to_a }.flatten)
+  end
+
+end

data/lib/eye/checker.rb

@@ -0,0 +1,138 @@
+class Eye::Checker
+  include Eye::Logger::Helpers
+
+  autoload :Memory,     'eye/checker/memory'
+  autoload :Cpu,        'eye/checker/cpu'
+  autoload :Http,       'eye/checker/http'
+  autoload :FileCTime,  'eye/checker/file_ctime'
+  autoload :FileSize,   'eye/checker/file_size'
+  autoload :Socket,     'eye/checker/socket'
+
+  TYPES = {:memory => "Memory", :cpu => "Cpu", :http => "Http", 
+           :ctime => "FileCTime", :fsize => "FileSize", :socket => "Socket"}
+
+  attr_accessor :value, :values, :options, :pid, :type, :check_count
+
+  def self.get_class(type)
+    klass = eval("Eye::Checker::#{TYPES[type]}") rescue nil
+    raise "Unknown checker #{type}" unless klass
+    klass
+  end
+
+  def self.create(pid, options = {}, logger_prefix = nil)
+    get_class(options[:type]).new(pid, options, logger_prefix)
+  end
+
+  def self.validate!(options)
+    get_class(options[:type]).validate(options)
+  end
+
+  def initialize(pid, options = {}, logger_prefix = nil)
+    @pid = pid
+    @options = options
+    @type = options[:type]
+
+    @logger = Eye::Logger.new(logger_prefix, "check:#{check_name}")
+    debug "create checker, with #{options}"
+    
+    @value = nil
+    @values = Eye::Utils::Tail.new(max_tries)
+    @check_count = 0
+  end
+
+  def last_human_values
+    h_values = @values.map do |v| 
+      sign = v[:good] ? '' : '*'
+      sign + human_value(v[:value]).to_s
+    end
+
+    '[' + h_values * ', ' + ']'
+  end
+
+  def check
+    @value = get_value_safe
+    @values << {:value => @value, :good => good?(value)}
+
+    result = true
+    @check_count += 1
+
+    if @values.size == max_tries
+      bad_count = @values.count{|v| !v[:good] }
+      result = false if bad_count >= min_tries
+    end
+
+    info "#{last_human_values} => #{result ? 'OK' : 'Fail'}"
+    result
+  end
+
+  def get_value_safe
+    get_value
+  end
+
+  def get_value
+    raise 'Realize me'
+  end
+
+  def human_value(value)
+    value.to_s
+  end
+
+  # true if check ok
+  # false if check bad
+  def good?(value)
+    raise 'Realize me'
+  end
+
+  def check_name
+    @check_name ||= @type.to_s
+  end
+
+  def max_tries
+    @max_tries ||= if times
+      if times.is_a?(Array)
+        times[-1].to_i
+      else
+        times.to_i
+      end
+    else
+      1
+    end    
+  end
+
+  def min_tries
+    @min_tries ||= if times
+      if times.is_a?(Array)
+        times[0].to_i
+      else
+        max_tries
+      end
+    else
+      max_tries
+    end
+  end
+
+  def previous_value
+    @values[-1][:value] if @values.present?
+  end
+
+  extend Eye::Dsl::Validation
+  param :every, [Fixnum, Float], false, 5
+  param :times, [Fixnum, Array]
+  param :fire, Symbol, nil, nil, [:stop, :restart, :unmonitor, :nothing]
+
+  class Defer < Eye::Checker
+    def get_value_safe
+      Celluloid::Future.new{ get_value }.value
+    end
+  end
+
+  class Custom < Defer
+    def self.inherited(base)
+      super
+      name = base.to_s
+      type = name.underscore.to_sym
+      Eye::Checker::TYPES[type] = name
+      Eye::Checker.const_set(name, base)
+    end
+  end
+end

data/lib/eye/checker/cpu.rb

@@ -0,0 +1,27 @@
+class Eye::Checker::Cpu < Eye::Checker
+
+  # checks :cpu, :every => 3.seconds, :below => 80, :times => [3,5]
+
+  param :below, [Fixnum, Float], true
+
+  def check_name
+    @check_name ||= "cpu(#{human_value(below)})"
+  end
+
+  def get_value
+    Eye::SystemResources.cpu(@pid).to_i # nil => 0
+  end
+
+  def human_value(value)
+    "#{value}%"
+  end
+
+  def good?(value)
+    if below
+      value < below
+    else
+      true
+    end
+  end
+
+end

data/lib/eye/checker/file_ctime.rb

@@ -0,0 +1,25 @@
+class Eye::Checker::FileCTime < Eye::Checker
+
+  # Check that file changes (log for example)
+
+  # checks :ctime, :every => 5.seconds, :file => "/tmp/1.log", :times => [3,5]
+
+  param :file, [String], true
+
+  def get_value
+    File.ctime(file) rescue nil
+  end
+
+  def human_value(value)
+    if value == nil
+      'Err'
+    else
+      value.strftime('%H:%M')
+    end
+  end
+
+  def good?(value)
+    value.to_i > previous_value.to_i
+  end
+
+end

data/lib/eye/checker/file_size.rb

@@ -0,0 +1,34 @@
+class Eye::Checker::FileSize < Eye::Checker
+
+  # Check that file size changed (log for example)
+
+  # checks :fsize, :every => 5.seconds, :file => "/tmp/1.log", :times => [3,5], 
+  #      :below => 30.kilobytes, :above => 10.kilobytes
+
+  param :file, [String], true
+  param :below, [Fixnum, Float]
+  param :above, [Fixnum, Float]
+
+  def get_value
+    File.size(file) rescue nil
+  end
+
+  def human_value(value)
+    "#{value.to_i / 1024}Kb"
+  end
+
+  def good?(value)
+    return true unless previous_value
+
+    diff = value.to_i - previous_value.to_i
+
+    return true if diff < 0 # case when logger nulled
+
+    return false if below && diff > below
+    return false if above && diff < above
+    return false if diff == 0
+
+    true
+  end
+
+end

data/lib/eye/checker/http.rb

@@ -0,0 +1,98 @@
+require 'net/http'
+
+class Eye::Checker::Http < Eye::Checker::Defer
+
+  # checks :http, :every => 5.seconds, :times => 1,
+  #  :url => "http://127.0.0.1:3000/", :kind => :success, :pattern => /OK/, :timeout => 3.seconds
+
+  param :url,           String, true
+  param :pattern,       [String, Regexp]
+  param :kind
+  param :timeout,       [Fixnum, Float]
+  param :open_timeout,  [Fixnum, Float]
+  param :read_timeout,  [Fixnum, Float]
+
+  attr_reader :uri
+
+  def initialize(*args)
+    super
+
+    @uri = URI.parse(url) rescue URI.parse('http://127.0.0.1')
+    @pattern = pattern
+    @kind = case kind
+              when Fixnum then Net::HTTPResponse::CODE_TO_OBJ[kind]
+              when String, Symbol then Net.const_get("HTTP#{kind.to_s.camelize}") rescue Net::HTTPSuccess
+            else
+              Net::HTTPSuccess
+            end
+    @open_timeout = (open_timeout || 3).to_f
+    @read_timeout = (read_timeout || timeout || 15).to_f
+  end
+  
+  def get_value
+    res = session.start{ |http| http.get(@uri.request_uri) }
+    {:result => res}
+
+  rescue Timeout::Error => ex
+    debug ex.inspect
+
+    if defined?(Net::OpenTimeout) # for ruby 2.0
+      mes = ex.class.is_a?(Net::OpenTimeout) ? "OpenTimeout<#{@open_timeout}>" : "ReadTimeout<#{@read_timeout}>"
+      {:exception => mes}
+    else
+      error "#{ex.message}"
+      {:exception => "Timeout<#{@open_timeout},#{@read_timeout}>"}
+    end
+
+  rescue => ex
+    {:exception => "Error<#{ex.message}>"}
+  end
+
+  def good?(value)
+    return false unless value[:result]
+
+    unless value[:result].kind_of?(@kind)
+      return false 
+    end
+
+    if @pattern
+      matched = if @pattern.is_a?(Regexp)
+        @pattern === value[:result].body
+      else
+        value[:result].body.include?(@pattern.to_s)
+      end
+      value[:notice] = "missing '#{@pattern.to_s}'" unless matched
+      matched
+    else
+      true
+    end
+  end
+
+  def human_value(value)
+    if !value.is_a?(Hash)
+      '-'
+    elsif value[:exception]
+      value[:exception]
+    else
+      body_size = value[:result].body.size / 1024
+      msg = "#{value[:result].code}=#{body_size}Kb"
+      msg += "<#{value[:notice]}>" if value[:notice]
+      msg
+    end
+  end  
+
+private
+
+  def session
+    Net::HTTP.new(@uri.host, @uri.port).tap do |session|
+      if @uri.scheme == 'https'
+        require 'net/https'
+        session.use_ssl = true
+        session.verify_mode = OpenSSL::SSL::VERIFY_NONE
+      end
+      session.open_timeout = @open_timeout
+      session.read_timeout = @read_timeout
+    end
+  end
+
+end