flapjack 0.6.37 → 0.6.38

data/.travis.yml

@@ -5,6 +5,6 @@ gemfile:
   - Gemfile
 services:
   - redis-server
-# uncomment this line if your project needs to run something other than `rake`:
-# script: bundle exec rspec spec
-
+before_script:
+  - mkdir -p ./log
+script: bundle exec rspec spec && bundle exec cucumber features

data/README.md

@@ -1,6 +1,11 @@
 Flapjack
 ========
 
+[![Travis CI Status][id_travis_img]][id_travis_link]
+
+[id_travis_link]: https://secure.travis-ci.org/#!/flpjck/flapjack
+[id_travis_img]: https://secure.travis-ci.org/flpjck/flapjack.png
+
 Flapjack is a highly scalable and distributed monitoring notification system.
 
 Flapjack provides a scalable method for dealing with events representing changes in system state (OK -> WARNING -> CRITICAL transitions) and alerting appropriate people as necessary.

data/lib/flapjack/coordinator.rb

@@ -21,6 +21,7 @@ require 'flapjack/oobetet'
 require 'flapjack/pagerduty'
 require 'flapjack/notification/email'
 require 'flapjack/notification/sms'
+require 'flapjack/redis_pool'
 require 'flapjack/web'
 
 module Flapjack
@@ -158,12 +159,13 @@ module Flapjack
 
       port = 3001 if (port.nil? || port <= 0 || port > 65535)
 
-      pikelet_class.class_variable_set('@@redis', build_redis_connection_pool)
+      pool = Flapjack::RedisPool.new(:config => @redis_options)
+      pikelet_class.class_variable_set('@@redis', pool)
 
       Thin::Logging.silent = true
 
       pikelet = Thin::Server.new('0.0.0.0', port, pikelet_class, :signals => false)
-      @pikelets << {:instance => pikelet, :type => pikelet_type}
+      @pikelets << {:instance => pikelet, :type => pikelet_type, :pool => pool}
       pikelet.start
       @logger.debug "new thin server instance started for #{pikelet_type}"
     end
@@ -179,8 +181,10 @@ module Flapjack
 
       # set up connection pooling, stop resque errors (ensure that it's only
       # done once)
+      pool = nil
       if (['email_notifier', 'sms_notifier'] & @pikelets.collect {|p| p[:type]}).empty?
-        ::Resque.redis = build_redis_connection_pool
+        pool = Flapjack::RedisPool.new(:config => @redis_options)
+        ::Resque.redis = pool
         ## NB: can override the default 'resque' namespace like this
         #::Resque.redis.namespace = 'flapjack'
       end
@@ -221,17 +225,13 @@ module Flapjack
           stop
         end
       }
-      @pikelets << {:fiber => f, :type => pikelet_type}
+      pikelet_values = {:fiber => f, :type => pikelet_type}
+      pikelet_values[:pool] = pool if pool
+      @pikelets << pikelet_values
       f.resume
       @logger.debug "new fiber created for #{pikelet_type}"
     end
 
-    def build_redis_connection_pool(options = {})
-      EventMachine::Synchrony::ConnectionPool.new(:size => options[:size] || 5) do
-        ::Redis.new(@redis_options)
-      end
-    end
-
     # # TODO rewrite to be less spammy -- print only initial state and changes
     # def health_check
     #   @pikelets.each do |pik|
@@ -259,10 +259,8 @@ module Flapjack
             }.resume
           end
         when EM::Resque::Worker
-          if pik[:fiber] && pik[:fiber].alive?
-            # resque is polling, so we don't need a shutdown object
-            pik[:instance].shutdown
-          end
+          # resque is polling, so we don't need a shutdown object
+          pik[:instance].shutdown if pik[:fiber] && pik[:fiber].alive?
         when Thin::Server # web, api
           # drop from this side, as HTTP keepalive etc. means browsers
           # keep connections alive for ages, and we'd be hanging around
@@ -280,6 +278,16 @@ module Flapjack
           if fibers.any?(&:alive?) || thin_pikelets.any?{|tp| !tp.backend.empty? }
             EM::Synchrony.sleep 0.25
           else
+            @pikelets.select {|p| thin_pikelets.include?(p) }.each do |tp|
+              tp[:pool].empty!
+            end
+
+            rsq_p = @pikelets.detect {|p|
+              ['email_notifier', 'sms_notifier'].include?(p[:type]) && !p[:pool].nil?
+            }
+
+            rsq_p[:pool].empty! if rsq_p
+
             EM.stop
             break
           end

data/lib/flapjack/data/contact.rb

@@ -3,6 +3,8 @@
 # NB: use of redis.keys probably indicates we should maintain a data
 # structure to avoid the need for this type of query
 
+require 'set'
+
 module Flapjack
 
   module Data
@@ -85,18 +87,35 @@ module Flapjack
           merge('service_key' => service_key)
       end
 
-      def entities
-        @redis.keys('contacts_for:*').inject([]) {|ret, k|
+      def entities_and_checks
+        @redis.keys('contacts_for:*').inject({}) {|ret, k|
           if @redis.sismember(k, self.id)
-            k =~ /^contacts_for:(.+)$/
-            entity_id = $1
-            if entity_name = @redis.hget("entity:#{entity_id}", 'name')
-              ret << Flapjack::Data::Entity.new(:name => entity_name,
-                      :id => entity_id, :redis => @redis)
+            if k =~ /^contacts_for:([a-zA-Z0-9][a-zA-Z0-9\.\-]*[a-zA-Z0-9])(?::(\w+))?$/
+              entity_id = $1
+              check = $2
+
+              unless ret.has_key?(entity_id)
+                ret[entity_id] = {}
+                if entity_name = @redis.hget("entity:#{entity_id}", 'name')
+                  entity = Flapjack::Data::Entity.new(:name => entity_name,
+                             :id => entity_id, :redis => @redis)
+                  ret[entity_id][:entity] = entity
+                end
+                # using a set to ensure unique check values
+                ret[entity_id][:checks] = Set.new
+              end
+
+              if check
+                # just add this check for the entity
+                ret[entity_id][:checks] |= check
+              else
+                # registered for the entity so add all checks
+                ret[entity_id][:checks] |= entity.check_list
+              end
             end
           end
           ret
-        }
+        }.values
       end
 
       def name

data/lib/flapjack/executive.rb

@@ -45,7 +45,10 @@ module Flapjack
       @filters << Flapjack::Filters::Delays.new(options)
       @filters << Flapjack::Filters::Acknowledgement.new(options)
 
-      @boot_time = Time.now
+      @boot_time    = Time.now
+      @fqdn         = `/bin/hostname -f`.chomp
+      @pid          = Process.pid
+      @instance_id  = "#{@fqdn}:#{@pid}"
 
       # FIXME: all of the below keys assume there is only ever one executive running;
       # we could generate a fuid and save it to disk, and prepend it from that
@@ -61,6 +64,12 @@ module Flapjack
         @redis.hset('event_counters', 'failure', 0)
         @redis.hset('event_counters', 'action', 0)
       end
+
+      @redis.zadd('executive_instances', @boot_time.to_i, @instance_id)
+      @redis.hset("event_counters:#{@instance_id}", 'all', 0)
+      @redis.hset("event_counters:#{@instance_id}", 'ok', 0)
+      @redis.hset("event_counters:#{@instance_id}", 'failure', 0)
+      @redis.hset("event_counters:#{@instance_id}", 'action', 0)
     end
 
     def main
@@ -73,6 +82,9 @@ module Flapjack
         event = Flapjack::Data::Event.next(:persistence => @redis)
         process_event(event) unless event.nil?
       end
+
+      @redis.empty! if @redis
+
       @logger.info("Exiting main loop.")
     end
 
@@ -123,6 +135,7 @@ module Flapjack
       result    = { :skip_filters => false }
       timestamp = Time.now.to_i
       @event_count = @redis.hincrby('event_counters', 'all', 1)
+      @event_count = @redis.hincrby("event_counters:#{@instance_id}", 'all', 1)
 
       # FIXME skip if entity_check.nil?
 
@@ -137,8 +150,10 @@ module Flapjack
 
         if event.ok?
           @redis.hincrby('event_counters', 'ok', 1)
+          @redis.hincrby("event_counters:#{@instance_id}", 'ok', 1)
         elsif event.failure?
           @redis.hincrby('event_counters', 'failure', 1)
+          @redis.hincrby("event_counters:#{@instance_id}", 'failure', 1)
           @redis.hset('unacknowledged_failures', @event_count, event.id)
         end
 
@@ -167,7 +182,8 @@ module Flapjack
       when 'action'
         # When an action event is processed, store the event.
         @redis.hset(event.id + ':actions', timestamp, event.state)
-        @redis.hincrby('event_counters', 'action', 1) if event.ok?
+        @redis.hincrby('event_counters', 'action', 1)
+        @redis.hincrby("event_counters:#{@instance_id}", 'action', 1)
 
         if event.acknowledgement? && event.acknowledgement_id
           @redis.hdel('unacknowledged_failures', event.acknowledgement_id)

data/lib/flapjack/jabber.rb

@@ -315,6 +315,9 @@ module Flapjack
 
       count_timer.cancel
       keepalive_timer.cancel
+
+      @redis.empty! if @redis
+      @redis_handler.empty! if @redis_handler
     end
 
   end

data/lib/flapjack/oobetet.rb

@@ -3,19 +3,12 @@
 #require 'socket'
 
 require 'eventmachine'
-# the redis/synchrony gems need to be required in this particular order, see                                                # the redis-rb README for details
-#require 'hiredis'
 require 'em-synchrony'
-#require 'redis/connection/synchrony'
-#require 'redis'
-
-#require 'chronic_duration'
 
 require 'blather/client/client'
 require 'em-synchrony/fiber_iterator'
 require 'yajl/json_gem'
 
-#require 'flapjack/data/entity_check'
 require 'flapjack/pikelet'
 require 'flapjack/utility'
 

data/lib/flapjack/pagerduty.rb

@@ -223,6 +223,9 @@ module Flapjack
       end
 
       acknowledgement_timer.cancel
+
+      @redis.empty! if @redis
+      @redis_timer.empty! if @redis_timer
     end
 
   end

data/lib/flapjack/pikelet.rb

@@ -12,6 +12,8 @@ require 'log4r'
 require 'log4r/outputter/consoleoutputters'
 require 'log4r/outputter/syslogoutputter'
 
+require 'flapjack/redis_pool'
+
 module Flapjack
   module Pikelet
     attr_accessor :logger, :redis, :config
@@ -27,9 +29,7 @@ module Flapjack
     def build_redis_connection_pool(options = {})
       return unless @bootstrapped
       if defined?(EventMachine) && defined?(EventMachine::Synchrony)
-        EventMachine::Synchrony::ConnectionPool.new(:size => options[:size] || 5) do
-          ::Redis.new(@redis_config)
-        end
+        Flapjack::RedisPool.new(:config => @redis_config, :size => options[:size])
       else
         ::Redis.new(@redis_config)
       end

data/lib/flapjack/redis_pool.rb

@@ -0,0 +1,40 @@
+#!/usr/bin/env ruby
+
+require 'eventmachine'
+# the redis/synchrony gems need to be required in this particular order, see
+# the redis-rb README for details
+require 'hiredis'
+require 'em-synchrony'
+require 'redis/connection/synchrony'
+require 'redis'
+
+# require 'eventmachine/synchrony/connection_pool'
+
+module Flapjack
+  class RedisPool < EventMachine::Synchrony::ConnectionPool
+
+    def initialize(opts = {})
+      config = opts.delete(:config)
+      super(:size => opts[:size] || 5) {
+        ::Redis.new(config)
+      }
+    end
+
+    def empty!
+      f = Fiber.current
+
+      until @available.empty? && @pending.empty?
+        begin
+          conn = acquire(f)
+          conn.quit
+          @available.delete(conn)
+        ensure
+          if pending = @pending.shift
+            pending.resume
+          end
+        end
+      end
+    end
+
+  end
+end

data/lib/flapjack/version.rb

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
 
 module Flapjack
-  VERSION = "0.6.37"
+  VERSION = "0.6.38"
 end

data/lib/flapjack/web.rb

@@ -196,7 +196,9 @@ module Flapjack
         @pagerduty_credentials = @contact.pagerduty_credentials
       end
 
-      @entities = @contact.entities
+      @entities_and_checks = @contact.entities_and_checks.sort_by {|ec|
+        ec[:entity].name
+      }
 
       haml :contact
     end
@@ -232,18 +234,21 @@ module Flapjack
     end
 
     def self_stats
+      @fqdn         = `/bin/hostname -f`.chomp
+      @pid          = Process.pid
+      @instance_id  = "#{@fqdn}:#{@pid}"
+
       @keys = @@redis.keys '*'
-      @count_failing_checks  = @@redis.zcard 'failed_checks'
-      @count_all_checks      = @@redis.keys('check:*:*').length
-      @event_counter_all     = @@redis.hget('event_counters', 'all')
-      @event_counter_ok      = @@redis.hget('event_counters', 'ok')
-      @event_counter_failure = @@redis.hget('event_counters', 'failure')
-      @event_counter_action  = @@redis.hget('event_counters', 'action')
-      @boot_time             = Time.at(@@redis.get('boot_time').to_i)
-      @uptime                = Time.now.to_i - @boot_time.to_i
-      @uptime_string         = time_period_in_words(@uptime)
-      @event_rate_all        = (@uptime > 0) ?
-                                 (@event_counter_all.to_f / @uptime) : 0
+      @count_failing_checks    = @@redis.zcard 'failed_checks'
+      @count_all_checks        = @@redis.keys('check:*:*').length
+      @executive_instances     = @@redis.zrange('executive_instances', '0', '-1', :withscores => true)
+      @event_counters          = @@redis.hgetall('event_counters')
+      @event_counters_instance = @@redis.hgetall("event_counters:#{@instance_id}")
+      @boot_time               = Time.at(@@redis.zscore('executive_instances', @instance_id).to_i)
+      @uptime                  = Time.now.to_i - @boot_time.to_i
+      @uptime_string           = time_period_in_words(@uptime)
+      @event_rate_all          = (@uptime > 0) ?
+                                 (@event_counters_instance['all'].to_f / @uptime) : 0
       @events_queued = @@redis.llen('events')
     end
 

data/lib/flapjack/web/views/contact.haml

@@ -11,40 +11,36 @@
     %div#wrapper
       = nav
       %h1= @contact.name
-      - if @entities && !@entities.empty?
+      - if @contact.media && !@contact.media.empty?
+        %ul
+          - @contact.media.each_pair do |mk, mv|
+            - if 'pagerduty'.eql?(mk)
+              %li= "PagerDuty: "
+              %ul
+                - @pagerduty_credentials.each_pair do |pk, pv|
+                  - if 'password'.eql?(pk)
+                    %li= "#{pk}: ..."
+                  - else
+                    %li= "#{pk}: #{pv}"
+            - else
+              %li= "#{mk.capitalize}: #{mv}"
+      - else
+        %p No media
+      - if @entities_and_checks && !@entities_and_checks.empty?
         %table
           %tr
             %th Entity name
             %th Checks
-            %th Media
-          - @entities.sort_by(&:name).each do |entity|
+          - @entities_and_checks.each do |ec|
+            - entity = ec[:entity]
+            - checks = ec[:checks]
             %tr
               %td
                 %p= entity.name
               %td
-                - checks = entity.check_list
-                - if !checks.empty?
-                  - checks.each do |check|
-                    - link = "/check?entity=#{entity.name}&check=#{check}"
-                    %p
-                      %a(title='check status' href=link) #{check}
-                - else
-                  %p No checks
-              %td
-                - if @contact.media && !@contact.media.empty?
-                  %ul
-                    - @contact.media.each_pair do |mk, mv|
-                      - if 'pagerduty'.eql?(mk)
-                        %li= "PagerDuty: "
-                        %ul
-                          - @pagerduty_credentials.each_pair do |pk, pv|
-                            - if 'password'.eql?(pk)
-                              %li= "#{pk}: ..."
-                            - else
-                              %li= "#{pk}: #{pv}"
-                      - else
-                        %li= "#{mk.capitalize}: #{mv}"
-                - else
-                  %p No media
+                - checks.each do |check|
+                  - link = "/check?entity=#{entity.name}&check=#{check}"
+                  %p
+                    %a(title='check status' href=link) #{check}
       - else
         %p No entities

data/lib/flapjack/web/views/self_stats.haml

@@ -17,9 +17,11 @@
       %h4
         Number of failing services:
         = @count
-      %p Events processed: #{@event_counter_all} (ok: #{@event_counter_ok}, failure: #{@event_counter_failure}, action: #{@event_counter_action})
-      %p Average rate: #{@event_rate_all} events per second
+      %p Events processed (all time): #{@event_counters['all']} (ok: #{@event_counters['ok']}, failure: #{@event_counters['failure']}, action: #{@event_counters['action']})
+      %p Events processed (this instance): #{@event_counters_instance['all']} (ok: #{@event_counters_instance['ok']}, failure: #{@event_counters_instance['failure']}, action: #{@event_counters_instance['action']})
+      %p Average rate (this instance): #{@event_rate_all} events per second
       %p Total keys in redis: #{@keys.length}
       %p Uptime: #{@uptime_string}
       %p Boot time: #{@boot_time}
       %p Current time: #{Time.now}
+      %p Executive Instances: #{@executive_instances.inspect}

data/spec/lib/flapjack/executive_spec.rb

@@ -36,6 +36,7 @@ describe Flapjack::Executive, :redis => true do
 
     executive = Flapjack::Executive.new
     executive.bootstrap(:config => {})
+    @redis.should_receive(:empty!)
     executive.should_receive(:build_redis_connection_pool).and_return(@redis)
 
     # hacky, but the behaviour it's mimicking (shutdown from another thread) isn't

data/spec/lib/flapjack/jabber_spec.rb

@@ -134,6 +134,7 @@ describe Flapjack::Jabber do
     EM::Synchrony.should_receive(:add_periodic_timer).with(60).and_return(timer_2)
 
     redis = mock('redis')
+    redis.should_receive(:empty!)
 
     fj = Flapjack::Jabber.new
     fj.bootstrap(:config => config)

data/spec/lib/flapjack/web_spec.rb

@@ -25,11 +25,9 @@ describe Flapjack::Web, :sinatra => true, :redis => true do
     @redis.should_receive(:keys).with('*').and_return([])
     @redis.should_receive(:zcard).with('failed_checks')
     @redis.should_receive(:keys).with('check:*:*').and_return([])
-    @redis.should_receive(:hget).with('event_counters', 'all')
-    @redis.should_receive(:hget).with('event_counters', 'ok')
-    @redis.should_receive(:hget).with('event_counters', 'failure')
-    @redis.should_receive(:hget).with('event_counters', 'action')
-    @redis.should_receive(:get).with('boot_time').and_return(0)
+    @redis.should_receive(:zscore).with('executive_instances', anything).and_return(Time.now.to_i)
+    @redis.should_receive(:hgetall).twice.and_return({'all' => '8001', 'ok' => '8002'},
+      {'all' => '9001', 'ok' => '9002'}) #
     @redis.should_receive(:llen).with('events')
   end
 
@@ -67,6 +65,7 @@ describe Flapjack::Web, :sinatra => true, :redis => true do
   end
 
   it "shows a page listing failing checks" do
+    @redis.should_receive(:zrange).with("executive_instances", "0", "-1", :withscores => true)
     @redis.should_receive(:zrange).with('failed_checks', 0, -1).and_return(["#{entity_name}:#{check}:states"])
 
     expect_stats

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: flapjack
 version: !ruby/object:Gem::Version
-  version: 0.6.37
+  version: 0.6.38
   prerelease: 
 platform: ruby
 authors:
@@ -11,7 +11,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-03 00:00:00.000000000 Z
+date: 2012-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: daemons
@@ -453,6 +453,7 @@ files:
 - lib/flapjack/persistence/sqlite3.rb
 - lib/flapjack/persistence/sqlite3/sqlite3.rb
 - lib/flapjack/pikelet.rb
+- lib/flapjack/redis_pool.rb
 - lib/flapjack/transports/beanstalkd.rb
 - lib/flapjack/transports/result.rb
 - lib/flapjack/utility.rb