flapjack 0.7.34 → 0.7.35

data/CHANGELOG.md

@@ -1,5 +1,12 @@
 ## Flapjack Changelog
 
+# 0.7.35 - 2013-12-10
+- Feature: allow flapper to flap with an arbitrary interval gh-383 (@jessereynolds)
+- Feature: Expose statistics for currency of all checks gh-386 (@jessereynolds)
+- Chore: WebUI: contacts page - move notification rules section higher gh-376 (@jessereynolds)
+- Bug: 500 errors in api are not logged by default gh-379 (@ali-graham)
+- Bug: Exception generating notifications when state_duration is nil gh-372 (@jessereynolds)
+
 # 0.7.34 - 2013-11-20
 - Feature: update logoage (@jessereynolds)
 - Bug: flapjack-nagios-receiver to start daemonized from init script (@jessereynolds)

data/Gemfile

@@ -11,6 +11,7 @@ group :test do
   gem 'cucumber'
   gem 'delorean'
   gem 'rack-test'
+  gem 'async_rack_test'
   gem 'resque_spec'
   gem 'webmock'
   gem 'guard'

data/bin/flapper

@@ -25,13 +25,14 @@ ensure
   Socket.do_not_reverse_lookup = orig
 end
 
-def main(bind_ip)
-  start_every = 120
-  stop_after  = 60
-  bind_port   = 12345
+def main(bind_ip, bind_port, frequency)
+  raise "bind_port must be an integer" unless bind_port.is_a?(Integer)
+  start_every = frequency
+  stop_after  = frequency.to_f / 2
+
   EM.run {
 
-    puts "#{Time.now}: starting server"
+    puts "#{Time.now}: starting server on #{bind_ip}:#{bind_port}"
     server_init = EM.start_server bind_ip, bind_port, Flapper
     EM.add_timer(stop_after) do
       puts "#{Time.now}: stopping server"
@@ -39,7 +40,7 @@ def main(bind_ip)
     end
 
     EM.add_periodic_timer(start_every) do
-      puts "#{Time.now}: starting server"
+      puts "#{Time.now}: starting server on #{bind_ip}:#{bind_port}"
       server = EM.start_server bind_ip, bind_port, Flapper
       EM.add_timer(stop_after) do
         puts "#{Time.now}: stopping server"
@@ -82,12 +83,22 @@ OptionParser.new do |opts|
     options.bind_ip = b
   end
 
+  opts.on("-P", "--bind-port [PORT]", String, "PORT for flapper to bind to") do |p|
+    options.bind_port = p.to_i
+  end
+
+  opts.on("-f", "--frequency [SECONDS]", String, "oscillate with a frequency of SECONDS [120]") do |f|
+    options.frequency = f.to_f
+  end
+
 end.parse!(ARGV)
 
-pidfile = options.pidfile  || "/var/run/flapjack/#{exe}.pid"
-logfile = options.log_path || "/var/log/flapjack/#{exe}.log"
 daemonize = options.daemonize.nil? ? true : options.daemonize
-bind_ip = options.bind_ip  || local_ip
+pidfile   = options.pidfile   || "/var/run/flapjack/#{exe}.pid"
+logfile   = options.log_path  || "/var/log/flapjack/#{exe}.log"
+bind_ip   = options.bind_ip   || local_ip
+bind_port = options.bind_port || 12345
+frequency = options.frequency || 120.0
 
 runner = Dante::Runner.new(exe, :pid_path => pidfile, :log_path => logfile)
 
@@ -99,7 +110,7 @@ when "start"
   else
     print "#{exe} starting..."
     runner.execute(:daemonize => daemonize) {
-      main(bind_ip)
+      main(bind_ip, bind_port, frequency)
     }
     puts " done."
   end
@@ -117,7 +128,7 @@ when "stop"
 when "restart"
   print "#{exe} restarting..."
   runner.execute(:daemonize => true, :restart => true) {
-    main(bind_ip)
+    main(bind_ip, bind_port, frequency)
   }
   puts " done."
 

data/lib/flapjack/data/alert.rb

@@ -84,8 +84,10 @@ module Flapjack
         raise "state #{@state.inspect} is invalid" unless
           allowed_states.include?(@state)
 
-        raise "state_duration #{@state_duration.inspect} is invalid" unless
-          @state_duration && @state_duration.is_a?(Integer) && @state_duration >= 0
+        if @state_duration
+          raise "state_duration (#{@state_duration.inspect}) is invalid" unless
+            @state_duration.is_a?(Integer) && @state_duration >= 0
+        end
 
         if @rollup_alerts
           raise "rollup_alerts should be nil or a hash" unless @rollup_alerts.is_a?(Hash)

data/lib/flapjack/data/entity.rb

@@ -116,6 +116,16 @@ module Flapjack
         Flapjack::Data::EntityCheck.find_all_failing_by_entity(:redis => redis).keys
       end
 
+      def self.find_all_current(options)
+        raise "Redis connection not set" unless redis = options[:redis]
+        redis.zrange('current_entities', 0, -1)
+      end
+
+      def self.find_all_current_with_last_update(options)
+        raise "Redis connection not set" unless redis = options[:redis]
+        redis.zrange('current_entities', 0, -1, {:withscores => true})
+      end
+
       def contacts
         contact_ids = @redis.smembers("contacts_for:#{id}")
 

data/lib/flapjack/data/entity_check.rb

@@ -130,6 +130,80 @@ module Flapjack
         redis.hget("check:#{event_id}", 'state')
       end
 
+      # takes an array of ages (in seconds) to split all checks up by
+      # - age means how long since the last update
+      # - 0 age is implied if not explicitly passed
+      # returns arrays of all current checks hashed by age range upper bound, eg:
+      #
+      # EntityCheck.find_all_split_by_freshness([60, 300], opts) =>
+      #   {   0 => [ 'foo-app-01:SSH' ],
+      #      60 => [ 'foo-app-01:Ping', 'foo-app-01:Disk / Utilisation' ],
+      #     300 => [] }
+      #
+      # you can also set :counts to true in options and you'll just get the counts, eg:
+      #
+      # EntityCheck.find_all_split_by_freshness([60, 300], opts.merge(:counts => true)) =>
+      #   {   0 => 1,
+      #      60 => 3,
+      #     300 => 0 }
+      #
+      # and you can get the last update time with each check too by passing :with_times => true eg:
+      #
+      # EntityCheck.find_all_split_by_freshness([60, 300], opts.merge(:with_times => true)) =>
+      #   {   0 => [ ['foo-app-01:SSH', 1382329923.0] ],
+      #      60 => [ ['foo-app-01:Ping', 1382329922.0], ['foo-app-01:Disk / Utilisation', 1382329921.0] ],
+      #     300 => [] }
+      #
+      def self.find_all_split_by_freshness(ages, options)
+        raise "Redis connection not set" unless redis = options[:redis]
+
+        raise "ages does not respond_to? :each and :each_with_index" unless ages.respond_to?(:each) && ages.respond_to?(:each_with_index)
+        raise "age values must respond_to? :to_i" unless ages.all? {|age| age.respond_to?(:to_i) }
+
+        ages << 0
+        ages = ages.sort.uniq
+
+        start_time = Time.now
+
+        checks = []
+        # get all the current checks, with last update time
+        Flapjack::Data::Entity.find_all_current(:redis => redis).each do |entity|
+          redis.zrange("current_checks:#{entity}", 0, -1, {:withscores => true}).each do |check|
+            check[0] = "#{entity}:#{check[0]}"
+            checks << check
+          end
+        end
+
+        skeleton = ages.inject({}) {|memo, age| memo[age] = [] ; memo }
+        age_ranges = ages.reverse.each_cons(2)
+        results_with_times = checks.inject(skeleton) do |memo, check|
+          check_age = start_time.to_i - check[1]
+          check_age = 0 unless check_age > 0
+          if check_age >= ages.last
+            memo[ages.last] << check
+          else
+            age_range = age_ranges.detect {|a, b| check_age < a && check_age >= b }
+            memo[age_range.last] << check unless age_range.nil?
+          end
+          memo
+        end
+
+        case
+        when options[:with_times]
+          results_with_times
+        when options[:counts]
+          results_with_times.inject({}) do |memo, (age, checks)|
+            memo[age] = checks.length
+            memo
+          end
+        else
+          results_with_times.inject({}) do |memo, (age, checks)|
+            memo[age] = checks.map { |check| check[0] }
+            memo
+          end
+        end
+      end
+
       def entity_name
         entity.name
       end

data/lib/flapjack/gateways/api.rb

@@ -29,16 +29,42 @@ module Flapjack
 
       set :show_exceptions, false
 
-      #rescue_exception = Proc.new { |env, exception|
-      #  @logger.error exception.message
-      #  @logger.error exception.backtrace.join("\n")
-      #  [503, {}, {:errors => [exception.message]}.to_json]
-      #}
-      #use Rack::FiberPool, :size => 25, :rescue_exception => rescue_exception
-      #
-      # FIXME: not sure why the above isn't working, had to add a general
-      # error handler later in this file
-      use Rack::FiberPool, :size => 25
+      rescue_exception = Proc.new { |env, exception|
+
+        error = proc {|status, exception, *msg|
+          if !msg || msg.empty?
+            trace = exception.backtrace.join("\n")
+            msg = "#{exception.class} - #{exception.message}"
+            msg_str = "#{msg}\n#{trace}"
+          else
+            msg_str = msg.join(", ")
+          end
+          logger = Flapjack::Gateways::API.instance_variable_get('@logger')
+          case
+          when status < 500
+            logger.warn "Error: #{msg_str}"
+          else
+            logger.error "Error: #{msg_str}"
+          end
+          [status, {}, {:errors => msg}.to_json]
+        }
+
+        e = env['sinatra.error']
+
+        case exception
+        when Flapjack::Gateways::API::ContactNotFound
+          error.call(403, e, "could not find contact '#{e.contact_id}'")
+        when Flapjack::Gateways::API::NotificationRuleNotFound
+          error.call(403, e, "could not find notification rule '#{e.rule_id}'")
+        when Flapjack::Gateways::API::EntityNotFound
+          error.call(403, e, "could not find entity '#{e.entity}'")
+        when Flapjack::Gateways::API::EntityCheckNotFound
+          error.call(403, e, "could not find entity check '#{e.check}'")
+        else
+          error.call(500, exception)
+        end
+      }
+      use Rack::FiberPool, :size => 25, :rescue_exception => rescue_exception
 
       use Rack::MethodOverride
       use Rack::JsonParamsParser
@@ -65,11 +91,16 @@ module Flapjack
       end
 
       before do
-        input = env['rack.input'].read
-        input_short = input.gsub(/\n/, '').gsub(/\s+/, ' ')
-        logger.info("#{request.request_method} #{request.path_info}#{request.query_string} #{input_short[0..80]}")
-        logger.debug("#{request.request_method} #{request.path_info}#{request.query_string} #{input}")
-        env['rack.input'].rewind
+        input = nil
+        if logger.debug?
+          input = env['rack.input'].read
+          logger.debug("#{request.request_method} #{request.path_info}#{request.query_string} #{input}")
+        elsif logger.info?
+          input = env['rack.input'].read
+          input_short = input.gsub(/\n/, '').gsub(/\s+/, ' ')
+          logger.info("#{request.request_method} #{request.path_info}#{request.query_string} #{input_short[0..80]}")
+        end
+        env['rack.input'].rewind unless input.nil?
       end
 
       after do
@@ -81,35 +112,9 @@ module Flapjack
       register Flapjack::Gateways::API::ContactMethods
 
       not_found do
-        logger.debug("in not_found :-(")
         err(404, "not routable")
       end
 
-      error Flapjack::Gateways::API::ContactNotFound do
-        e = env['sinatra.error']
-        err(403, "could not find contact '#{e.contact_id}'")
-      end
-
-      error Flapjack::Gateways::API::NotificationRuleNotFound do
-        e = env['sinatra.error']
-        err(403, "could not find notification rule '#{e.rule_id}'")
-      end
-
-      error Flapjack::Gateways::API::EntityNotFound do
-        e = env['sinatra.error']
-        err(403, "could not find entity '#{e.entity}'")
-      end
-
-      error Flapjack::Gateways::API::EntityCheckNotFound do
-        e = env['sinatra.error']
-        err(403, "could not find entity check '#{e.check}'")
-      end
-
-      error do
-        e = env['sinatra.error']
-        err(response.status, "#{e.class} - #{e.message}")
-      end
-
       private
 
       def err(status, *msg)
@@ -117,7 +122,6 @@ module Flapjack
         logger.info "Error: #{msg_str}"
         [status, {}, {:errors => msg}.to_json]
       end
-
     end
 
   end

data/lib/flapjack/gateways/web.rb

@@ -132,11 +132,11 @@ module Flapjack
         entity_stats
         check_stats
         {
-          'events_queued'    => @events_queued,
-          'all_entities'     => @count_all_entities,
-          'failing_entities' => @count_failing_entities,
-          'all_checks'       => @count_all_checks,
-          'failing_checks'   => @count_failing_checks,
+          'events_queued'       => @events_queued,
+          'all_entities'        => @count_all_entities,
+          'failing_entities'    => @count_failing_entities,
+          'all_checks'          => @count_all_checks,
+          'failing_checks'      => @count_failing_checks,
           'processed_events' => {
             'all_time' => {
               'total'   => @event_counters['all'].to_i,
@@ -145,10 +145,11 @@ module Flapjack
               'action'  => @event_counters['action'].to_i,
             }
           },
-          'total_keys' => @dbsize,
-          'uptime'     => @uptime_string,
-          'boottime'   => @boot_time,
-          'current_time' => Time.now,
+          'check_freshness'     => @current_checks_ages,
+          'total_keys'          => @dbsize,
+          'uptime'              => @uptime_string,
+          'boottime'            => @boot_time,
+          'current_time'        => Time.now,
           'executive_instances' => @executive_instances,
         }.to_json
       end
@@ -283,7 +284,6 @@ module Flapjack
       end
 
       get '/contacts' do
-        #self_stats
         @contacts = Flapjack::Data::Contact.all(:redis => redis)
 
         erb 'contacts.html'.to_sym
@@ -394,6 +394,7 @@ module Flapjack
         end
         @event_counters = redis.hgetall('event_counters')
         @events_queued  = redis.llen('events')
+        @current_checks_ages = Flapjack::Data::EntityCheck.find_all_split_by_freshness([0, 60, 300, 900, 3600], {:redis => redis, :counts => true } )
       end
 
       def entity_stats

data/lib/flapjack/gateways/web/views/contact.html.erb

@@ -106,32 +106,6 @@
           </table>
         <% end %>
 
-        <h3>All Entities and Checks</h3>
-        <% if !@entities_and_checks || @entities_and_checks.empty? %>
-          <p>No entities</p>
-        <% else %>
-          <table class="table table-bordered table-hover table-condensed">
-            <tr>
-              <th>Entity</th>
-              <th>Checks</th>
-            </tr>
-            <% @entities_and_checks.each do |ec| %>
-              <%
-                entity = ec[:entity]
-                checks = ec[:checks]
-               %>
-              <tr>
-                <td><a href="/entity/<%= u(entity.name) %>" title="entity status"><%= h entity.name %></a></td>
-                <td>
-                  <% checks.each do |check| %>
-                  <%=   "<a href=\"/check?entity=#{u(entity.name)}&amp;check=#{u(check)}\" title=\"check status\">#{ h check }</a>" %>
-                  <% end %>
-                </td>
-              </tr>
-            <% end %>
-          </table>
-        <% end %>
-
         <h3>Notification Rules</h3>
         <% rules = @contact.notification_rules %>
         <% if !rules || rules.empty? %>
@@ -161,7 +135,33 @@
                 <td><%= h(blackholes.join(', ')) %></td>
               </tr>
             <% end %>
-            </table>
+          </table>
+        <% end %>
+
+        <h3>All Entities and Checks</h3>
+        <% if !@entities_and_checks || @entities_and_checks.empty? %>
+          <p>No entities</p>
+        <% else %>
+          <table class="table table-bordered table-hover table-condensed">
+            <tr>
+              <th>Entity</th>
+              <th>Checks</th>
+            </tr>
+            <% @entities_and_checks.each do |ec| %>
+              <%
+                entity = ec[:entity]
+                checks = ec[:checks]
+               %>
+              <tr>
+                <td><a href="/entity/<%= u(entity.name) %>" title="entity status"><%= h entity.name %></a></td>
+                <td>
+                  <% checks.each do |check| %>
+                  <%=   "<a href=\"/check?entity=#{u(entity.name)}&amp;check=#{u(check)}\" title=\"check status\">#{ h check }</a>" %>
+                  <% end %>
+                </td>
+              </tr>
+            <% end %>
+          </table>
         <% end %>
 
       </div>

data/lib/flapjack/gateways/web/views/self_stats.html.erb

@@ -50,6 +50,16 @@
                   </ul>
                 </td>
               </tr>
+              <tr>
+                <td>Check Freshness</td>
+                <td>
+                  <ul>
+                    <% @current_checks_ages.each_pair do |age, check_count| %>
+                      <li>&gt;= <%= age %>: <%= check_count %></li>
+                    <% end %>
+                  </ul>
+                </td>
+              </tr>
               <tr>
                 <td>Total keys in redis</td>
                 <td><%= h @dbsize %></td>