eye 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ec5135b6c531e9451f98ec8b9d36a54d2e57e3f9
-  data.tar.gz: fd8011e705ee96d0d24dffbe03be598a18b285c4
+  metadata.gz: b27fc1f2b1ca918d1dec48f63af0f59446f6a166
+  data.tar.gz: b823fadfaea2111a8c57ec07e4b2ae84ad7899a2
 SHA512:
-  metadata.gz: 59237be9adb524bdc90ddb9203d3246e7e6fc03f4b9c7398df2380fba2423e68592a996ac720ff04cd8a813b34ed8a27a1c259421907dd1103b18ad508c9decd
-  data.tar.gz: b741a332f8888e3f5a2dd50af91e0dbafebed8f8dcc406f9f8cab270b565f3ce2624adef2e4b9c2b3be0a01ff18dcbaf28159d4ba55b6d532e37711b8c9fa5cd
+  metadata.gz: 6fd472f37e8123f426b45409e2067c1bf901cf265aa37a3480229eda6dcbc8fcdca4c51656580cc756c305da9f565f18a35dda277a48f188da36bb69a02effa0
+  data.tar.gz: ce0459d1f8d2d0d236a9ca2586ed44e2e173b6c82bfc6c426c473e9d0c20dce20bf28935af89efc23f2e33a48be9953a61fd0a2e0b5b536ef62e8fbd32a38997

data/README.md

@@ -30,7 +30,7 @@ Eye.application "test" do
   working_dir File.expand_path(File.join(File.dirname(__FILE__), %w[ processes ]))
   stdall "trash.log" # stdout,err logs for processes by default
   env "APP_ENV" => "production" # global env for each processes
-  triggers :flapping, :times => 10, :within => 1.minute # 
+  triggers :flapping, :times => 10, :within => 1.minute, :retry_in => 10.minutes
   checks :cpu, :below => 100, :times => 3 # global check for all processes
 
   group "samples" do

data/examples/test.eye

@@ -14,7 +14,7 @@ Eye.application "test" do
   working_dir File.expand_path(File.join(File.dirname(__FILE__), %w[ processes ]))
   stdall "trash.log" # stdout,err logs for processes by default
   env "APP_ENV" => "production" # global env for each processes
-  triggers :flapping, :times => 10, :within => 1.minute # 
+  triggers :flapping, :times => 10, :within => 1.minute, :retry_in => 10.minutes
   checks :cpu, :below => 100, :times => 3 # global check for all processes
 
   group "samples" do

data/eye.gemspec

@@ -8,7 +8,7 @@ Gem::Specification.new do |gem|
     %q{Process monitoring tool. An alternative to God and Bluepill. With Bluepill like config syntax. Requires MRI Ruby >= 1.9.2. Uses Celluloid and Celluloid::IO.}
   gem.homepage      = "http://github.com/kostya/eye"
 
-  gem.files         = `git ls-files`.split($\)
+  gem.files         = `git ls-files`.split($\).reject{|n| n =~ %r[png|gif\z]}
   gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.name          = "eye"
@@ -21,7 +21,7 @@ Gem::Specification.new do |gem|
   
   gem.add_dependency 'celluloid',     '~> 0.12.0'
   gem.add_dependency 'celluloid-io',  '~> 0.12.0'
-  gem.add_dependency 'state_machine'
+  gem.add_dependency 'state_machine', '< 1.2'
   gem.add_dependency 'activesupport', '~> 3.2.0'
   gem.add_dependency 'thor'
 

data/lib/eye.rb

@@ -1,5 +1,5 @@
 module Eye  
-  VERSION   = "0.2.2"
+  VERSION   = "0.2.3"
   ABOUT     = "Eye v#{VERSION} (c) 2012-2013 @kostya"
 
   autoload :Process,        'eye/process'

data/lib/eye/checker/http.rb

@@ -95,7 +95,7 @@ private
     Net::HTTP.new(@uri.host, @uri.port).tap do |session|
       if @uri.scheme == 'https'
         require 'net/https'
-        session.use_ssl=true
+        session.use_ssl = true
         session.verify_mode = OpenSSL::SSL::VERIFY_NONE
       end
       session.open_timeout = @open_timeout

data/lib/eye/controller.rb

@@ -39,7 +39,7 @@ class Eye::Controller
 
     Eye::SystemResources.setup
     
-    info "starting #{Eye::ABOUT}"
+    info "starting #{Eye::ABOUT} (#{$$})"
   end
 
 end

data/lib/eye/controller/send_command.rb

@@ -90,7 +90,7 @@ private
 
     res = []
     str = Regexp.escape(mask).gsub('\*', '.*?')
-    r = %r{\A#{str}\z}
+    r = %r{\A#{str}}
 
     # find app
     res = @applications.select{|a| a.name =~ r || a.full_name =~ r }

data/lib/eye/loader.rb

@@ -6,7 +6,7 @@ gem 'nio4r'
 gem 'facter'
 gem 'timers'
 
-gem 'state_machine'
+gem 'state_machine', '< 1.2'
 gem 'activesupport', '~> 3.2.0'
 
 gem 'i18n' # for as

data/lib/eye/process.rb

@@ -18,7 +18,7 @@ class Eye::Process
   autoload :Validate,         'eye/process/validate'
 
   attr_accessor :pid, :watchers, :config, :states_history, 
-                :childs, :triggers, :flapping, :name
+                :childs, :triggers, :name, :state_reason
   
   def initialize(config)
     raise 'pid file should be' unless config[:pid_file]
@@ -29,7 +29,6 @@ class Eye::Process
     @watchers = {}
     @childs = {}
     @triggers = []
-    @flapping = false
     @name = @config[:name]
 
     @states_history = Eye::Process::StatesHistory.new(100)

data/lib/eye/process/data.rb

@@ -39,7 +39,8 @@ module Eye::Process::Data
   end
 
   def sub_object?(obj)
-    # we not recognize childs
+    return false if self.class == Eye::ChildProcess
+    self.childs.each { |_, child| return true if child == obj }
     false
   end
 

data/lib/eye/process/monitor.rb

@@ -76,13 +76,15 @@ private
 
   def check_crash
     if down?
-      if self[:keep_alive] && !@flapping
+      if self[:keep_alive]
         warn 'check crashed: process is down'
         schedule :start, 'crashed'
       else
-        warn 'check crashed: process is down, and flapping happens (or not keep_alive option)'
-        schedule :unmonitor, 'flapping'
+        warn 'check crashed: process without keep_alive'
+        schedule :unmonitor, 'crashed'
       end
+    else
+      debug 'check crashed: skipped, process is not in down'
     end
   end
 

data/lib/eye/process/scheduler.rb

@@ -3,7 +3,7 @@ module Eye::Process::Scheduler
   # ex: schedule :update_config, config, "reason: update_config"
   def schedule(command, *args, &block)
     if scheduler.alive?
-      unless self.respond_to?(command)
+      unless self.respond_to?(command, true)
         warn "object not support :#{command} to schedule"
         return
       end
@@ -17,6 +17,14 @@ module Eye::Process::Scheduler
     end
   end
 
+  def schedule_in(interval, command, *args, &block)
+    debug "schedule_in #{interval} :#{command} #{args}"
+    after(interval.to_f) do
+      debug "scheduled_in #{interval} :#{command} #{args}"
+      schedule(command, *args, &block)
+    end
+  end
+
   def scheduled_action(command, h = {}, &block)
     reason = h.delete(:reason)
     info "=> #{command} #{h[:args].present? ? "#{h[:args]*',' }" : nil} #{reason ? "(reason: #{reason})" : nil}"

data/lib/eye/process/states.rb

@@ -52,12 +52,12 @@ class Eye::Process
       transition any => :unmonitored
     end
 
-    after_transition :on => :crashed, :do => :on_crashed
     after_transition any => :unmonitored, :do => :on_unmonitored
     after_transition any-:up => :up, :do => :on_up
     after_transition :up => any-:up, :do => :from_up
     after_transition any => any, :do => :log_transition
-    after_transition any => any, :do => :upd_for_triggers
+    after_transition any => any, :do => :check_triggers
+    after_transition :on => :crashed, :do => :on_crashed
   end
 
   def on_crashed
@@ -65,7 +65,6 @@ class Eye::Process
   end
 
   def on_unmonitored
-    self.flapping = false
     self.pid = nil
   end
 
@@ -84,8 +83,4 @@ class Eye::Process
     info "switch :#{transition.event} [:#{transition.from_name} => :#{transition.to_name}] #{@state_reason ? "(reason: #{@state_reason})" : nil}"
   end
 
-  def upd_for_triggers(transition)
-    check_triggers
-  end
-
 end

data/lib/eye/process/states_history.rb

@@ -8,11 +8,11 @@ class Eye::Process::StatesHistory < Eye::Utils::Tail
     self.map{|c| c[:state] }
   end
 
-  def states_for_period(period)
+  def states_for_period(period, from_time = nil)
     tm = Time.now - period
-    self.select do |s|
-      s[:at] >= tm
-    end.map{|c| c[:state] }
+    tm = [tm, from_time].max if from_time
+
+    self.select{|s| s[:at] >= tm }.map{|c| c[:state] }
   end
 
   def last_state

data/lib/eye/process/trigger.rb

@@ -16,9 +16,8 @@ module Eye::Process::Trigger
     return if unmonitored?
 
     self.triggers.each do |trigger|
-      if !trigger.check(self.states_history)
-        notify :crit, 'flapping!'
-        @flapping = true        
+      unless trigger.check(self.states_history)
+        on_flapping(trigger) if trigger.class == Eye::Trigger::Flapping
       end
     end
   end
@@ -26,7 +25,30 @@ module Eye::Process::Trigger
 private
 
   def add_trigger(cfg = {})
-    self.triggers << Eye::Trigger.create(cfg, logger.prefix)
+    trigger = Eye::Trigger.create(cfg, logger.prefix)
+    self.triggers << trigger
   end
 
-end
+  def on_flapping(trigger)
+    notify :crit, 'flapping!'
+    schedule :unmonitor, "flapping"
+
+    @retry_times ||= 0
+    retry_in = trigger.retry_in
+
+    return unless retry_in
+    return if trigger.retry_times && @retry_times >= trigger.retry_times
+
+    schedule_in(retry_in.to_f, :retry_action)
+  end
+
+  def retry_action
+    debug "trigger retry timer"
+    return unless unmonitored?
+    return unless state_reason.to_s.include?('flapping') # TODO: remove hackety
+
+    schedule :start, "retry start after flapping"
+    @retry_times += 1
+  end
+
+end

data/lib/eye/system_resources.rb

@@ -74,7 +74,7 @@ class Eye::SystemResources
   private
 
     def set
-      @ps_aux = Eye::System.ps_aux
+      @ps_aux = defer{ Eye::System.ps_aux }
       @at = Time.now
     end
 

data/lib/eye/trigger.rb

@@ -27,7 +27,7 @@ class Eye::Trigger
     @options = options
     @logger = Eye::Logger.new(logger_prefix, "trigger")
 
-    debug "add trigger #{options}"
+    debug "add #{options}"
   end
 
   def check(states_history)

data/lib/eye/trigger/flapping.rb

@@ -1,20 +1,24 @@
 class Eye::Trigger::Flapping < Eye::Trigger
 
-  # triggers :flapping, :times => 10, :within => 1.minute
+  # triggers :flapping, :times => 10, :within => 1.minute, 
+  #          :retry_in => 10.minutes, :retry_times => 15
 
-  param :times, [Fixnum], true
+  param :times, [Fixnum], true, 5
   param :within, [Float, Fixnum], true
+  param :retry_in, [Float, Fixnum]
+  param :retry_times, [Fixnum]
 
-  def good?
-    return true unless within
-
-    states = @states_history.states_for_period( within )
+  def initialize(*args)
+    super
+    @last_at = nil
+  end
 
-    starting_count = states.count{|st| st == :starting}
-    down_count = states.count{|st| st == :down}
-    times_count = times || 5
+  def good?
+    states = @states_history.states_for_period( within, @last_at )
+    down_count = states.count{|st| st == :down }
 
-    if (starting_count >= times_count) && (down_count >= times_count)
+    if down_count >= times
+      @last_at = @states_history.last_state_changed_at
       false
     else
       true

data/spec/controller/find_objects_spec.rb

@@ -77,6 +77,26 @@ describe "find_objects" do
     subject.find_objects("asdfasdf").should == []
   end
 
+  describe "submatching without * " do
+    it "match by start symbols, apps" do
+      objs = subject.find_objects("app")
+      objs.map{|c| c.class}.should == [Eye::Application, Eye::Application]
+      objs.map{|c| c.name}.sort.should == %w{app1 app2}
+    end
+
+    it "match by start symbols, groups" do
+      objs = subject.find_objects("gr")
+      objs.map{|c| c.class}.should == [Eye::Group, Eye::Group]
+      objs.map{|c| c.name}.sort.should == %w{gr1 gr2}
+    end
+
+    it "match by start symbols, process" do
+      objs = subject.find_objects("z")
+      objs.map{|c| c.class}.should == [Eye::Process]
+      objs.map{|c| c.name}.sort.should == %w{z1}
+    end
+  end
+
   describe "find by routes" do
     it "group" do
       objs = subject.find_objects("app1:gr2")
@@ -102,7 +122,7 @@ describe "find_objects" do
       subject{ c = Eye::Controller.new; c.load(fixture("dsl/load_dubls.eye")); c }
 
       it "not found" do
-        subject.find_objects("z").should == []
+        subject.find_objects("zu").should == []
       end
 
       it "found 2 processed" do
@@ -139,11 +159,11 @@ describe "find_objects" do
 
   describe "missing" do
     it "should not found" do
-      subject.find_objects("app1:gr2:q").should == []
-      subject.find_objects("gr1:p1").should == []
+      subject.find_objects("app1:gr4").should == []
+      subject.find_objects("gr:p").should == []
       subject.find_objects("pp1").should == []
       subject.find_objects("app1::").should == []
-      subject.find_objects("app1:").should == []
+      subject.find_objects("app1:=").should == []
     end
   end
 

data/spec/process/scheduler_spec.rb

@@ -155,4 +155,14 @@ describe "Scheduler" do
       @process.test3.should == [1, :bla, 3]
     end
   end
+
+  describe "schedule_in" do
+    it "should schedule to future" do
+      @process.schedule_in(1.second, :scheduler_test3, 1, 2, 3)
+      sleep 0.5
+      @process.test3.should == nil
+      sleep 0.6
+      @process.test3.should == [1,2,3]
+    end
+  end
 end

data/spec/process/states_history_spec.rb

@@ -26,6 +26,12 @@ describe "Eye::Process::StatesHistory" do
     @h.states_for_period(1.5.minutes).should == [:up, :down]
     @h.states_for_period(2.5.minutes).should == [:stop, :up, :down]
     @h.states_for_period(6.minutes).should == [:up, :down, :start, :stop, :up, :down]
+
+    # with start_point
+    @h.states_for_period(2.5.minutes, 5.minutes.ago).should == [:stop, :up, :down]
+    @h.states_for_period(2.5.minutes, nil).should == [:stop, :up, :down]
+    @h.states_for_period(2.5.minutes, 1.5.minutes.ago).should == [:up, :down]
+    @h.states_for_period(2.5.minutes, Time.now).should == []
   end
 
   it "seq?" do

data/spec/process/triggers/flapping_spec.rb

@@ -42,6 +42,7 @@ describe "Flapping" do
 
     @process.state_name.should == :unmonitored
     @process.watchers.keys.should == []
+    @process.states_history.states.last(2).should == [:down, :unmonitored]
   end
 
   it "process flapping emulate with kill" do
@@ -49,14 +50,37 @@ describe "Flapping" do
 
     @process.start
 
-    # 4 times because, flapping flag, check on next switch
-    4.times do
+    3.times do
       die_process!(@process.pid)
       sleep 3
     end
 
     @process.state_name.should == :unmonitored
     @process.watchers.keys.should == []
+
+    # ! should switched to unmonitored from down status
+    @process.states_history.states.last(2).should == [:down, :unmonitored]
+  end
+
+  it "process flapping, and then send to start and fast kill, should ok started" do
+    @process = process(@c.merge(:triggers => C.flapping(:times => 3, :within => 15)))
+
+    @process.start
+
+    3.times do
+      die_process!(@process.pid)
+      sleep 3
+    end
+
+    @process.state_name.should == :unmonitored
+    @process.watchers.keys.should == []
+
+    @process.start
+    @process.state_name.should == :up
+
+    die_process!(@process.pid)
+    sleep 4
+    @process.state_name.should == :up
   end
 
   it "flapping not happens" do
@@ -67,15 +91,152 @@ describe "Flapping" do
     proxy(@process).schedule(:check_crash, anything)
     dont_allow(@process).schedule(:unmonitor)
 
+    sleep 2
 
-    sleep 5
-
-    # even if process die in middle
-    die_process!(@process.pid)
+    2.times do
+      die_process!(@process.pid)
+      sleep 3
+    end
 
-    sleep 5
+    sleep 2
 
     @process.state_name.should == :up    
   end
 
+  describe "retry_in, retry_times" do
+    before :each do
+      @c = C.p1.merge(
+        :triggers => C.flapping(:times => 2, :within => 3),
+        :start_grace => 0.1, # for fast flapping
+        :stop_grace => 0,
+        :start_command => @c[:start_command] + " -r"
+      )    
+    end
+
+    it "flapping than wait for interval and try again" do
+      @process = process(@c.merge(:triggers => C.flapping(:times => 2, :within => 3, 
+        :retry_in => 5.seconds)))
+      @process.start!
+
+      sleep 18
+
+      h = @process.states_history
+
+      # был в unmonitored
+      h.shift[:state].should == :unmonitored
+
+      # должен попытаться подняться два раза,
+      h.shift(6)
+
+      # затем перейти в unmonitored с причиной flapping
+      flapp1 = h.shift
+      flapp1[:state].should == :unmonitored
+      flapp1[:reason].should == 'flapping'
+
+      # затем снова попыться подняться два раза
+      h.shift(6)
+
+      # и снова перейти в unmonitored с причиной flapping
+      flapp2 = h.shift
+      flapp2[:state].should == :unmonitored
+      flapp2[:reason].should == 'flapping'
+
+      # интервал между переходами во flapping должен быть больше 8 сек
+      (flapp2[:at] - flapp1[:at]).should > 5.seconds
+
+      # тут снова должен пытаться подниматься так как нет лимитов
+      h.should_not be_blank
+    end
+
+    it "flapping retry 1 times with retry_times = 1" do
+      @process = process(@c.merge(:triggers => C.flapping(:times => 2, :within => 3, 
+        :retry_in => 5.seconds, :retry_times => 1)))
+      @process.start!
+
+      sleep 18
+
+      h = @process.states_history
+
+      # был в unmonitored
+      h.shift[:state].should == :unmonitored
+
+      # должен попытаться подняться два раза,
+      h.shift(6)
+
+      # затем перейти в unmonitored с причиной flapping
+      flapp1 = h.shift
+      flapp1[:state].should == :unmonitored
+      flapp1[:reason].should == 'flapping'
+
+      # затем снова попыться подняться два раза
+      h.shift(6)
+
+      # и снова перейти в unmonitored с причиной flapping
+      flapp2 = h.shift
+      flapp2[:state].should == :unmonitored
+      flapp2[:reason].should == 'flapping'
+
+      # интервал между переходами во flapping должен быть больше 8 сек
+      (flapp2[:at] - flapp1[:at]).should > 5.seconds
+
+      # все финал
+      h.should be_blank
+    end
+
+    it "flapping than manually doing something, should not retry" do
+      @process = process(@c.merge(:triggers => C.flapping(:times => 2, :within => 3, 
+        :retry_in => 5.seconds)))
+      @process.start!
+
+      sleep 6
+      @process.send_command :unmonitor
+      sleep 9
+
+      h = @process.states_history
+
+      # был в unmonitored
+      h.shift[:state].should == :unmonitored
+
+      # должен попытаться подняться два раза,
+      h.shift(6)
+
+      # затем перейти в unmonitored с причиной flapping
+      flapp1 = h.shift
+      flapp1[:state].should == :unmonitored
+      flapp1[:reason].should == 'flapping'
+
+      # затем его руками переводят в unmonitored
+      unm = h.shift
+      unm[:state].should == :unmonitored
+      unm[:reason].should == 'unmonitor by user'
+
+      # все финал
+      h.should be_blank
+    end
+
+    it "without retry_in" do
+      @process = process(@c.merge(:triggers => C.flapping(:times => 2, :within => 3)))
+      @process.start!
+
+      sleep 10
+
+      h = @process.states_history
+
+      # был в unmonitored
+      h.shift[:state].should == :unmonitored
+
+      # должен попытаться подняться два раза,
+      h.shift(6)
+
+      # затем перейти в unmonitored с причиной flapping
+      flapp1 = h.shift
+      flapp1[:state].should == :unmonitored
+      flapp1[:reason].should == 'flapping'
+
+      # все финал
+      h.should be_blank
+    end
+
+  end
+
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: eye
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - Konstantin Makarchev
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-30 00:00:00.000000000 Z
+date: 2013-04-14 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: celluloid
@@ -42,16 +42,16 @@ dependencies:
   name: state_machine
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - <
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.2'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - '>='
+    - - <
       - !ruby/object:Gem::Version
-        version: '0'
+        version: '1.2'
 - !ruby/object:Gem::Dependency
   name: activesupport
   requirement: !ruby/object:Gem::Requirement