ace-eye 0.6.1

data/examples/rbenv.eye

@@ -0,0 +1,11 @@
+Eye.application "rbenv_example" do
+  env 'RBENV_ROOT' => '/usr/local/rbenv', 'PATH' => "/usr/local/rbenv/shims:/usr/local/rbenv/bin:#{ENV['PATH']}"
+  working_dir File.expand_path(File.join(File.dirname(__FILE__), %w[ processes ]))
+
+  process "some_process" do
+    pid_file "some.pid"
+    start_command "ruby some.rb"
+    daemonize true
+    stdall "some.log"
+  end
+end

data/examples/sidekiq.eye

@@ -0,0 +1,23 @@
+# Example: how to run sidekiq daemon
+
+def sidekiq_process(proxy, name)
+  rails_env = proxy.env['RAILS_ENV']
+
+  proxy.process(name) do
+    start_command "ruby ./bin/sidekiq -e #{rails_env} -C ./config/sidekiq.#{rails_env}.yml"
+    pid_file "tmp/pids/#{name}.pid"
+    stdall "log/#{name}.log"
+    daemonize true
+    stop_signals [:QUIT, 5.seconds, :TERM, 5.seconds, :KILL]
+
+    check :cpu, :every => 30, :below => 100, :times => 5
+    check :memory, :every => 30, :below => 300.megabytes, :times => 5
+  end
+end
+
+Eye.application :sidekiq_test do
+  working_dir File.expand_path(File.join(File.dirname(__FILE__), %w[ processes ]))
+  env "RAILS_ENV" => 'production'
+
+  sidekiq_process self, :sidekiq
+end

data/examples/test.eye

@@ -0,0 +1,87 @@
+# load submodules, here just for example
+Eye.load('./eye/*.rb')
+
+# Eye self-configuration section
+Eye.config do
+  logger '/tmp/eye.log'
+end
+
+# Adding application
+Eye.application 'test' do
+  # All options inherits down to the config leafs.
+  # except `env`, which merging down
+
+  working_dir File.expand_path(File.join(File.dirname(__FILE__), %w[ processes ]))
+  stdall 'trash.log' # stdout,err logs for processes by default
+  env 'APP_ENV' => 'production' # global env for each processes
+  trigger :flapping, times: 10, within: 1.minute, retry_in: 10.minutes
+  check :cpu, every: 10.seconds, below: 100, times: 3 # global check for all processes
+
+  group 'samples' do
+    chain grace: 5.seconds # chained start-restart with 5s interval, one by one.
+
+    # eye daemonized process
+    process :sample1 do
+      pid_file '1.pid' # pid_path will be expanded with the working_dir
+      start_command 'ruby ./sample.rb'
+
+      # when no stop_command or stop_signals, default stop is [:TERM, 0.5, :KILL]
+      # default `restart` command is `stop; start`
+
+      daemonize true
+      stdall 'sample1.log'
+
+      # ensure the CPU is below 30% at least 3 out of the last 5 times checked
+      check :cpu, below: 30, times: [3, 5]
+    end
+
+    # self daemonized process
+    process :sample2 do
+      pid_file '2.pid'
+      start_command 'ruby ./sample.rb -d --pid 2.pid --log sample2.log'
+      stop_command 'kill -9 {PID}'
+
+      # ensure the memory is below 300Mb the last 3 times checked
+      check :memory, every: 20.seconds, below: 300.megabytes, times: 3
+    end
+  end
+
+  # daemon with 3 children
+  process :forking do
+    pid_file 'forking.pid'
+    start_command 'ruby ./forking.rb start'
+    stop_command 'ruby forking.rb stop'
+    stdall 'forking.log'
+
+    start_timeout 10.seconds
+    stop_timeout 5.seconds
+
+    monitor_children do
+      restart_command 'kill -2 {PID}' # for this child process
+      check :memory, below: 300.megabytes, times: 3
+    end
+  end
+
+  # eventmachine process, daemonized with eye
+  process :event_machine do |p|
+    pid_file 'em.pid'
+    start_command 'ruby em.rb'
+    stdout 'em.log'
+    daemonize true
+    stop_signals [:QUIT, 2.seconds, :KILL]
+
+    check :socket, addr: 'tcp://127.0.0.1:33221', every: 10.seconds, times: 2,
+                   timeout: 1.second, send_data: 'ping', expect_data: /pong/
+  end
+
+  # thin process, self daemonized
+  process :thin do
+    pid_file 'thin.pid'
+    start_command 'bundle exec thin start -R thin.ru -p 33233 -d -l thin.log -P thin.pid'
+    stop_signals [:QUIT, 2.seconds, :TERM, 1.seconds, :KILL]
+
+    check :http, url: 'http://127.0.0.1:33233/hello', pattern: /World/,
+                 every: 5.seconds, times: [2, 3], timeout: 1.second
+  end
+
+end

data/examples/thin-farm.eye

@@ -0,0 +1,30 @@
+RUBY = 'ruby'
+BUNDLE = 'bundle'
+
+Eye.load("process_thin.rb")
+
+Eye.config do
+  logger "/tmp/eye.log"
+end
+
+Eye.app 'thin-farm' do
+  working_dir File.expand_path(File.join(File.dirname(__FILE__), %w[ processes ]))
+  env "RAILS_ENV" => "production"
+
+  stop_on_delete true # this option means, when we change pids and load config,
+                      # deleted processes will be stops
+
+  trigger :flapping, :times => 10, :within => 1.minute
+  check :memory, :below => 60.megabytes, :every => 30.seconds, :times => 5
+  start_timeout 30.seconds
+
+  group :web do
+    chain :action => :restart, :grace => 5.seconds
+    chain :action => :start, :grace => 0.2.seconds
+
+    (5555..5560).each do |port|
+      thin self, port
+    end
+  end
+
+end

data/examples/unicorn.eye

@@ -0,0 +1,39 @@
+# Example: now to run unicorn, and monitor its child processes
+
+RUBY = '/usr/local/ruby/1.9.3/bin/ruby' # ruby on the server
+RAILS_ENV = 'production'
+
+Eye.application "rails_unicorn" do
+  env "RAILS_ENV" => RAILS_ENV
+
+  # unicorn requires to be `ruby` in path (for soft restart)
+  env "PATH" => "#{File.dirname(RUBY)}:#{ENV['PATH']}"
+
+  working_dir File.expand_path(File.join(File.dirname(__FILE__), %w[ processes ]))
+
+  process("unicorn") do
+    pid_file "tmp/pids/unicorn.pid"
+    start_command "#{RUBY} ./bin/unicorn -Dc ./config/unicorn.rb -E #{RAILS_ENV}"
+    stdall "log/unicorn.log"
+
+    # stop signals:
+    # http://unicorn.bogomips.org/SIGNALS.html
+    stop_signals [:TERM, 10.seconds]
+
+    # soft restart
+    restart_command "kill -USR2 {PID}"
+
+    check :cpu, :every => 30, :below => 80, :times => 3
+    check :memory, :every => 30, :below => 150.megabytes, :times => [3,5]
+
+    start_timeout 100.seconds
+    restart_grace 30.seconds
+
+    monitor_children do
+      stop_command "kill -QUIT {PID}"
+      check :cpu, :every => 30, :below => 80, :times => 3
+      check :memory, :every => 30, :below => 150.megabytes, :times => [3,5]
+    end
+  end
+
+end

data/eye.gemspec

@@ -0,0 +1,40 @@
+require File.expand_path('../lib/eye', __FILE__)
+
+Gem::Specification.new do |gem|
+  gem.authors       = "Konstantin Makarchev"
+  gem.email         = "eye-rb@googlegroups.com"
+
+  gem.description   = gem.summary = \
+    %q{FORK FOR TESTING. Process monitoring tool. Inspired from Bluepill and God. Requires Ruby(MRI) >= 1.9.3-p194. Uses Celluloid and Celluloid::IO.}
+  gem.homepage      = "http://github.com/kostya/eye"
+
+  gem.files         = `git ls-files`.split($\).reject{|n| n =~ %r[png|gif\z]}.reject{|n| n =~ %r[^(test|spec|features)/]}
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  #gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.name          = "ace-eye"
+  gem.require_paths = ["lib"]
+  gem.version       = Eye::VERSION
+  gem.license       = "MIT"
+
+  gem.required_ruby_version     = '>= 1.9.2'
+  gem.required_rubygems_version = '>= 1.3.6'
+
+  gem.add_dependency 'celluloid',     '~> 0.15.0'
+  gem.add_dependency 'celluloid-io',  '~> 0.15.0'
+  gem.add_dependency 'state_machine'
+  gem.add_dependency 'thor'
+  gem.add_dependency 'sigar',         '~> 0.7.2'
+
+  gem.add_development_dependency 'rake'
+  gem.add_development_dependency 'rspec', '< 2.14'
+  gem.add_development_dependency 'rr'
+  gem.add_development_dependency 'ruby-graphviz'
+  gem.add_development_dependency 'forking'
+  gem.add_development_dependency 'fakeweb'
+  gem.add_development_dependency 'eventmachine', ">= 1.0.3"
+  gem.add_development_dependency 'sinatra'
+  gem.add_development_dependency 'thin'
+  gem.add_development_dependency 'xmpp4r'
+  gem.add_development_dependency 'coveralls'
+  gem.add_development_dependency 'simplecov', '>= 0.8.1'
+end

data/lib/eye.rb

@@ -0,0 +1,28 @@
+module Eye
+  VERSION   = "0.6.1"
+  ABOUT     = "Eye v#{VERSION} (c) 2012-2014 @kostya"
+  PROCLINE  = "eye monitoring v#{VERSION}"
+
+  autoload :Process,        'eye/process'
+  autoload :ChildProcess,   'eye/child_process'
+  autoload :Server,         'eye/server'
+  autoload :Logger,         'eye/logger'
+  autoload :System,         'eye/system'
+  autoload :SystemResources,'eye/system_resources'
+  autoload :Checker,        'eye/checker'
+  autoload :Trigger,        'eye/trigger'
+  autoload :Group,          'eye/group'
+  autoload :Dsl,            'eye/dsl'
+  autoload :Application,    'eye/application'
+  autoload :Local,          'eye/local'
+  autoload :Client,         'eye/client'
+  autoload :Utils,          'eye/utils'
+  autoload :Notify,         'eye/notify'
+  autoload :Config,         'eye/config'
+  autoload :Reason,         'eye/reason'
+  autoload :Sigar,          'eye/sigar'
+
+  autoload :Controller,     'eye/controller'
+  autoload :Control,        'eye/control'
+  autoload :Cli,            'eye/cli'
+end

data/lib/eye/application.rb

@@ -0,0 +1,73 @@
+class Eye::Application
+
+  attr_reader :groups, :name
+
+  def initialize(name, config = {})
+    @groups = Eye::Utils::AliveArray.new
+    @name = name
+    @config = config
+    debug 'created'
+  end
+
+  def logger_tag
+    full_name
+  end
+
+  def full_name
+    @name
+  end
+
+  def add_group(group)
+    @groups << group
+  end
+
+  # sort processes in name order
+  def resort_groups
+    @groups = @groups.sort { |a, b| a.hidden ? 1 : (b.hidden ? -1 : (a.name <=> b.name)) }
+  end
+
+  def status_data(debug = false)
+    h = { name: @name, type: :application, subtree: @groups.map{|gr| gr.status_data(debug) }}
+    h.merge!(debug: debug_data) if debug
+    h
+  end
+
+  def status_data_short
+    h = Hash.new
+    @groups.each do |c|
+      c.processes.each do |p|
+        h[p.state] ||= 0
+        h[p.state] += 1
+      end
+    end
+    { name: @name, type: :application, states: h}
+  end
+
+  def debug_data
+  end
+
+  def send_command(command, *args)
+    info "send_command #{command}"
+
+    @groups.each do |group|
+      group.send_command(command, *args)
+    end
+  end
+
+  def alive?
+    true # emulate celluloid actor method
+  end
+
+  def sub_object?(obj)
+    res = @groups.include?(obj)
+    res = @groups.any?{|gr| gr.sub_object?(obj)} if !res
+    res
+  end
+
+  def processes
+    out = []
+    @groups.each{|gr| out += gr.processes.to_a }
+    Eye::Utils::AliveArray.new(out)
+  end
+
+end

data/lib/eye/checker.rb

@@ -0,0 +1,258 @@
+class Eye::Checker
+  include Eye::Dsl::Validation
+
+  autoload :Memory,     'eye/checker/memory'
+  autoload :Cpu,        'eye/checker/cpu'
+  autoload :Http,       'eye/checker/http'
+  autoload :FileCTime,  'eye/checker/file_ctime'
+  autoload :FileSize,   'eye/checker/file_size'
+  autoload :FileTouched,'eye/checker/file_touched'
+  autoload :Socket,     'eye/checker/socket'
+  autoload :Nop,        'eye/checker/nop'
+  autoload :Runtime,    'eye/checker/runtime'
+  autoload :Cputime,    'eye/checker/cputime'
+  autoload :ChildrenCount, 'eye/checker/children_count'
+  autoload :ChildrenMemory,'eye/checker/children_memory'
+
+  TYPES = {:memory => 'Memory', :cpu => 'Cpu', :http => 'Http',
+           :ctime => 'FileCTime', :fsize => 'FileSize', :file_touched => 'FileTouched',
+           :socket => 'Socket', :nop => 'Nop', :runtime => 'Runtime', :cputime => 'Cputime',
+           :children_count => "ChildrenCount", :children_memory => "ChildrenMemory" }
+
+  attr_accessor :value, :values, :options, :pid, :type, :check_count, :process
+
+  param :every, [Fixnum, Float], false, 5
+  param :times, [Fixnum, Array], nil, 1
+  param :fires, [Symbol, Array], nil, nil, [:stop, :restart, :unmonitor, :start, :delete, :nothing, :notify]
+  param :initial_grace, [Fixnum, Float]
+  param :skip_initial_fails, [TrueClass, FalseClass]
+
+  def self.name_and_class(type)
+    type = type.to_sym
+    return {:name => type, :type => type} if TYPES[type]
+
+    if type =~ /\A(.*?)_?[0-9]+\z/
+      ctype = $1.to_sym
+      return {:name => type, :type => ctype} if TYPES[ctype]
+    end
+  end
+
+  def self.get_class(type)
+    klass = eval("Eye::Checker::#{TYPES[type]}") rescue nil
+    raise "Unknown checker #{type}" unless klass
+    if deps = klass.requires
+      Array(deps).each { |d| require d }
+    end
+    klass
+  end
+
+  def self.create(pid, options = {}, process = nil)
+    get_class(options[:type]).new(pid, options, process)
+
+  rescue Exception, Timeout::Error => ex
+    log_ex(ex)
+    nil
+  end
+
+  def self.validate!(options)
+    get_class(options[:type]).validate(options)
+  end
+
+  def initialize(pid, options = {}, process = nil)
+    @process = process
+    @pid = pid
+    @options = options.dup
+    @type = options[:type]
+    @full_name = @process.full_name if @process
+    @initialized_at = Time.now
+
+    debug "create checker, with #{options}"
+
+    @value = nil
+    @values = Eye::Utils::Tail.new(max_tries)
+    @check_count = 0
+  end
+
+  def inspect
+    "<#{self.class} @process='#{@full_name}' @options=#{@options} @pid=#{@pid}>"
+  end
+
+  def logger_tag
+    @process.logger.prefix
+  end
+
+  def logger_sub_tag
+    "check:#{check_name}"
+  end
+
+  def last_human_values
+    h_values = @values.map do |v|
+      sign = v[:good] ? '' : '*'
+      sign + human_value(v[:value]).to_s
+    end
+
+    '[' + h_values * ', ' + ']'
+  end
+
+  def check
+    if initial_grace && (Time.now - @initialized_at < initial_grace)
+      debug 'skipped initial grace'
+      return true
+    else
+      @options[:initial_grace] = nil
+    end
+
+    @value = get_value_safe
+    @good_value = good?(value)
+    @values << {:value => @value, :good => @good_value}
+
+    result = true
+    @check_count += 1
+
+    if @values.size == max_tries
+      bad_count = @values.count{|v| !v[:good] }
+      result = false if bad_count >= min_tries
+    end
+
+    if skip_initial_fails
+      if @good_value
+        @options[:skip_initial_fails] = nil
+      else
+        result = true
+      end
+    end
+
+    info "#{last_human_values} => #{result ? 'OK' : 'Fail'}"
+    result
+
+  rescue Exception, Timeout::Error => ex
+    log_ex(ex)
+  end
+
+  def get_value_safe
+    get_value
+  end
+
+  def get_value
+    raise NotImplementedError
+  end
+
+  def human_value(value)
+    value.to_s
+  end
+
+  # true if check ok
+  # false if check bad
+  def good?(value)
+    value
+  end
+
+  def check_name
+    @check_name ||= @type.to_s
+  end
+
+  def max_tries
+    @max_tries ||= if times
+      if times.is_a?(Array)
+        times[-1].to_i
+      else
+        times.to_i
+      end
+    else
+      1
+    end
+  end
+
+  def min_tries
+    @min_tries ||= if times
+      if times.is_a?(Array)
+        times[0].to_i
+      else
+        max_tries
+      end
+    else
+      max_tries
+    end
+  end
+
+  def previous_value
+    @values[-1][:value] if @values.present?
+  end
+
+  def run_in_process_context(p)
+    process.instance_exec(&p) if process.alive?
+  end
+
+  def fire
+    actions = fires ? Array(fires) : [:restart]
+    process.notify :warn, "Bounded #{check_name}: #{last_human_values} send to #{actions}"
+
+    actions.each do |action|
+      process.schedule action, Eye::Reason.new("bounded #{check_name}")
+    end
+  end
+
+  def defer(&block)
+    Celluloid::Future.new(&block).value
+  end
+
+  class Defer < Eye::Checker
+    def get_value_safe
+      Celluloid::Future.new{ get_value }.value
+    end
+  end
+
+  def self.register(base)
+    name = base.to_s.gsub('Eye::Checker::', '')
+    type = name.underscore.to_sym
+    Eye::Checker::TYPES[type] = name
+    Eye::Checker.const_set(name, base)
+  end
+
+  def self.requires
+  end
+
+  class CustomCell < Eye::Checker
+    def self.inherited(base)
+      super
+      register(base)
+    end
+  end
+
+  class Custom < Defer
+    def self.inherited(base)
+      super
+      register(base)
+    end
+  end
+
+  class CustomDefer < Defer
+    def self.inherited(base)
+      super
+      register(base)
+    end
+  end
+
+  class Measure < Eye::Checker
+    param :below, [Fixnum, Float]
+    param :above, [Fixnum, Float]
+
+    def good?(value)
+      return false if below && (value > below)
+      return false if above && (value < above)
+      true
+    end
+
+    def measure_str
+      if below && above
+        ">#{human_value(above)}<#{human_value(below)}"
+      elsif below
+        "<#{human_value(below)}"
+      elsif above
+        ">#{human_value(above)}"
+      else
+        '-'
+      end
+    end
+  end
+end