flapjack 0.6.38 → 0.6.39

data/lib/flapjack/pagerduty.rb

@@ -12,6 +12,7 @@ require 'redis'
 require 'yajl/json_gem'
 
 require 'flapjack/data/entity_check'
+require 'flapjack/data/global'
 require 'flapjack/pikelet'
 
 module Flapjack
@@ -20,144 +21,13 @@ module Flapjack
 
     include Flapjack::Pikelet
 
+    PAGERDUTY_EVENTS_API_URL   = 'https://events.pagerduty.com/generic/2010-04-15/create_event.json'
+    SEM_PAGERDUTY_ACKS_RUNNING = 'sem_pagerduty_acks_running'
+
     def setup
       @redis = build_redis_connection_pool
       logger.debug("New Pagerduty pikelet with the following options: #{@config.inspect}")
 
-      @pagerduty_events_api_url = 'https://events.pagerduty.com/generic/2010-04-15/create_event.json'
-      @pagerduty_acks_started = nil
-      @sem_pagerduty_acks_running = 'sem_pagerduty_acks_running'
-    end
-
-    def send_pagerduty_event(event)
-      options  = { :body => Yajl::Encoder.encode(event) }
-      http = EM::HttpRequest.new(@pagerduty_events_api_url).post(options)
-      response = Yajl::Parser.parse(http.response)
-      status   = http.response_header.status
-      logger.debug "send_pagerduty_event got a return code of #{status.to_s} - #{response.inspect}"
-      [status, response]
-    end
-
-    def test_pagerduty_connection
-      noop = { "service_key"  => "11111111111111111111111111111111",
-               "incident_key" => "Flapjack is running a NOOP",
-               "event_type"   => "nop",
-               "description"  => "I love APIs with noops." }
-      code, results = send_pagerduty_event(noop)
-      return true if code == 200 && results['status'] =~ /success/i
-      logger.error "Error: test_pagerduty_connection: API returned #{code.to_s} #{results.inspect}"
-      false
-    end
-
-    # this should be moved to a checks data model perhaps
-    def unacknowledged_failing_checks
-      failing_checks = @redis_timer.zrange('failed_checks', '0', '-1')
-      unless failing_checks.is_a?(Array)
-        @logger.error("redis.zrange returned something other than an array! Here it is: " + failing_checks.inspect)
-      end
-      ufc = failing_checks.reject {|check|
-        @redis_timer.exists(check + ':unscheduled_maintenance')
-      }
-      @logger.debug "found unacknowledged failing checks as follows: " + ufc.join(', ')
-      ufc
-    end
-
-    def pagerduty_acknowledged?(opts)
-      subdomain   = opts['subdomain']
-      username    = opts['username']
-      password    = opts['password']
-      check       = opts['check']
-
-      url = 'https://' + subdomain + '.pagerduty.com/api/v1/incidents'
-      query = { 'fields'       => 'incident_number,status,last_status_change_by',
-                'since'        => (Time.new.utc - (60*60*24*7)).iso8601,
-                'until'        => (Time.new.utc + (60*60*24)).iso8601,
-                'incident_key' => check,
-                'status'       => 'acknowledged' }
-
-      options = { :head  => { 'authorization' => [username, password] },
-                  :query => query }
-
-      http = EM::HttpRequest.new(url).get(options)
-      # DEBUG flapjack-pagerduty: pagerduty_acknowledged?: decoded response as:
-      # {"incidents"=>[{"incident_number"=>40, "status"=>"acknowledged",
-      # "last_status_change_by"=>{"id"=>"PO1NWPS", "name"=>"Jesse Reynolds",
-      # "email"=>"jesse@bulletproof.net",
-      # "html_url"=>"http://bltprf.pagerduty.com/users/PO1NWPS"}}], "limit"=>100, "offset"=>0,
-      # "total"=>1}
-      begin
-        response = Yajl::Parser.parse(http.response)
-      rescue Yajl::ParseError
-        @logger.error("failed to parse json from a post to #{url} ... response headers and body follows...")
-        @logger.error(http.response_header.inspect)
-        @logger.error(http.response)
-        return nil, nil
-      end
-      status   = http.response_header.status
-
-      @logger.debug("pagerduty_acknowledged?: decoded response as: #{response.inspect}")
-      if response.nil?
-        @logger.error('no valid response received from pagerduty!')
-        return nil, nil
-      end
-
-      if response['incidents'].nil?
-        @logger.error('no incidents found in response')
-        return nil, nil
-      end
-
-      if response['incidents'].length > 0
-        pg_acknowledged_by = response['incidents'].first['last_status_change_by']
-        return true, :pg_acknowledged_by => pg_acknowledged_by
-      else
-        return false, nil
-      end
-    end
-
-    def catch_pagerduty_acks
-
-      # ensure we're the only instance of the pagerduty acknowledgement check running (with a naive
-      # timeout of five minutes to guard against stale locks caused by crashing code) either in this
-      # process or in other processes
-      if (@pagerduty_acks_started and @pagerduty_acks_started > (Time.now.to_i - 300)) or
-          @redis_timer.get(@sem_pagerduty_acks_running) == 'true'
-        logger.debug("skipping looking for acks in pagerduty as this is already happening")
-        return
-      end
-
-      @pagerduty_acks_started = Time.now.to_i
-      @redis_timer.set(@sem_pagerduty_acks_running, 'true')
-      @redis_timer.expire(@sem_pagerduty_acks_running, 300)
-
-      logger.debug("looking for acks in pagerduty for unack'd problems")
-
-      # ok lets do it
-      unacknowledged_failing_checks.each {|check|
-        entity_check = Flapjack::Data::EntityCheck.for_event_id(check, { :redis => @redis_timer, :logger => @logger } )
-        pagerduty_credentials = entity_check.pagerduty_credentials(:redis => @redis_timer)
-
-        if pagerduty_credentials.length == 0
-          @logger.debug("Found no pagerduty creditials for #{entity_check.entity_name}:#{entity_check.check}, moving on")
-          next
-        end
-
-        # FIXME: try each set of credentials until one works (may have stale contacts turning up)
-        options = pagerduty_credentials.first.merge('check' => check)
-
-        pagerduty_acknowledged, result_hash = pagerduty_acknowledged?(options)
-        if pagerduty_acknowledged
-          pg_acknowledged_by = result_hash[:pg_acknowledged_by] unless result_hash.nil?
-          @logger.debug "#{check} is acknowledged in pagerduty, creating flapjack acknowledgement ... "
-          who_text = ""
-          if !pg_acknowledged_by.nil? && !pg_acknowledged_by['name'].nil?
-            who_text = " by #{pg_acknowledged_by['name']}"
-          end
-          entity_check.create_acknowledgement('summary' => "Acknowledged on PagerDuty" + who_text)
-        else
-          @logger.debug "#{check} is not acknowledged in pagerduty, moving on"
-        end
-      }
-      @redis_timer.del(@sem_pagerduty_acks_running)
       @pagerduty_acks_started = nil
     end
 
@@ -173,12 +43,12 @@ module Flapjack
       raise "Can't connect to the pagerduty API" unless test_pagerduty_connection
 
       # TODO: only clear this if there isn't another pagerduty gateway instance running
-      # or better, include on instance ID in the semaphore key name
-      @redis.del(@sem_pagerduty_acks_running)
+      # or better, include an instance ID in the semaphore key name
+      @redis.del(SEM_PAGERDUTY_ACKS_RUNNING)
 
       acknowledgement_timer = EM::Synchrony.add_periodic_timer(10) do
         @redis_timer ||= build_redis_connection_pool
-        catch_pagerduty_acks
+        find_pagerduty_acknowledgements_if_safe
       end
 
       queue = @config['queue']
@@ -190,9 +60,7 @@ module Flapjack
         event         = Yajl::Parser.parse(events[queue][1])
         type          = event['notification_type']
         logger.debug("pagerduty notification event popped off the queue: " + event.inspect)
-        if 'shutdown'.eql?(type)
-          # do anything in particular?
-        else
+        unless 'shutdown'.eql?(type)
           event_id      = event['event_id']
           entity, check = event_id.split(':')
           state         = event['state']
@@ -228,6 +96,138 @@ module Flapjack
       @redis_timer.empty! if @redis_timer
     end
 
+    # considering this as part of the public API -- exposes it for testing.
+    def find_pagerduty_acknowledgements_if_safe
+
+      # ensure we're the only instance of the pagerduty acknowledgement check running (with a naive
+      # timeout of five minutes to guard against stale locks caused by crashing code) either in this
+      # process or in other processes
+      if (@pagerduty_acks_started and @pagerduty_acks_started > (Time.now.to_i - 300)) or
+          @redis_timer.get(SEM_PAGERDUTY_ACKS_RUNNING) == 'true'
+        logger.debug("skipping looking for acks in pagerduty as this is already happening")
+        return
+      end
+
+      @pagerduty_acks_started = Time.now.to_i
+      @redis_timer.set(SEM_PAGERDUTY_ACKS_RUNNING, 'true')
+      @redis_timer.expire(SEM_PAGERDUTY_ACKS_RUNNING, 300)
+
+      find_pagerduty_acknowledgements
+
+      @redis_timer.del(SEM_PAGERDUTY_ACKS_RUNNING)
+      @pagerduty_acks_started = nil
+    end
+
+  private
+
+    def test_pagerduty_connection
+      noop = { "service_key"  => "11111111111111111111111111111111",
+               "incident_key" => "Flapjack is running a NOOP",
+               "event_type"   => "nop",
+               "description"  => "I love APIs with noops." }
+      code, results = send_pagerduty_event(noop)
+      return true if code == 200 && results['status'] =~ /success/i
+      logger.error "Error: test_pagerduty_connection: API returned #{code.to_s} #{results.inspect}"
+      false
+    end
+
+    def send_pagerduty_event(event)
+      options  = { :body => Yajl::Encoder.encode(event) }
+      http = EM::HttpRequest.new(PAGERDUTY_EVENTS_API_URL).post(options)
+      response = Yajl::Parser.parse(http.response)
+      status   = http.response_header.status
+      logger.debug "send_pagerduty_event got a return code of #{status.to_s} - #{response.inspect}"
+      [status, response]
+    end
+
+    def find_pagerduty_acknowledgements
+
+      logger.debug("looking for acks in pagerduty for unack'd problems")
+
+      unacknowledged_failing_checks = Flapjack::Data::Global.unacknowledged_failing_checks(:redis => @redis_timer)
+
+      @logger.debug "found unacknowledged failing checks as follows: " + unacknowledged_failing_checks.join(', ')
+
+      unacknowledged_failing_checks.each do |entity_check|
+        pagerduty_credentials = entity_check.pagerduty_credentials(:redis => @redis_timer)
+        check = entity_check.check
+
+        if pagerduty_credentials.empty?
+          @logger.debug("No pagerduty credentials found for #{entity_check.entity_name}:#{check}, skipping")
+          next
+        end
+
+        # FIXME: try each set of credentials until one works (may have stale contacts turning up)
+        options = pagerduty_credentials.first.merge('check' => check)
+
+        acknowledged = pagerduty_acknowledged?(options)
+        if acknowledged.nil?
+          @logger.debug "#{check} is not acknowledged in pagerduty, skipping"
+          next
+        end
+
+        pg_acknowledged_by = acknowledged[:pg_acknowledged_by]
+        @logger.debug "#{check} is acknowledged in pagerduty, creating flapjack acknowledgement... "
+        who_text = ""
+        if !pg_acknowledged_by.nil? && !pg_acknowledged_by['name'].nil?
+          who_text = " by #{pg_acknowledged_by['name']}"
+        end
+        entity_check.create_acknowledgement('summary' => "Acknowledged on PagerDuty" + who_text)
+      end
+
+    end
+
+    def pagerduty_acknowledged?(opts)
+      subdomain   = opts['subdomain']
+      username    = opts['username']
+      password    = opts['password']
+      check       = opts['check']
+
+      t = Time.now.utc
+
+      url = 'https://' + subdomain + '.pagerduty.com/api/v1/incidents'
+      query = { 'fields'       => 'incident_number,status,last_status_change_by',
+                'since'        => (t - (60*60*24*7)).iso8601, # the last week
+                'until'        => (t + (60*60*24)).iso8601,   # 1 day in the future
+                'incident_key' => check,
+                'status'       => 'acknowledged' }
+
+      options = { :head  => { 'authorization' => [username, password] },
+                  :query => query }
+
+      http = EM::HttpRequest.new(url).get(options)
+      # DEBUG flapjack-pagerduty: pagerduty_acknowledged?: decoded response as:
+      # {"incidents"=>[{"incident_number"=>40, "status"=>"acknowledged",
+      # "last_status_change_by"=>{"id"=>"PO1NWPS", "name"=>"Jesse Reynolds",
+      # "email"=>"jesse@bulletproof.net",
+      # "html_url"=>"http://bltprf.pagerduty.com/users/PO1NWPS"}}], "limit"=>100, "offset"=>0,
+      # "total"=>1}
+      begin
+        response = Yajl::Parser.parse(http.response)
+      rescue Yajl::ParseError
+        @logger.error("failed to parse json from a post to #{url} ... response headers and body follows...")
+        @logger.error(http.response_header.inspect)
+        @logger.error(http.response)
+        return nil
+      end
+      status   = http.response_header.status
+
+      @logger.debug("pagerduty_acknowledged?: decoded response as: #{response.inspect}")
+      if response.nil?
+        @logger.error('no valid response received from pagerduty!')
+        return nil
+      end
+
+      if response['incidents'].nil?
+        @logger.error('no incidents found in response')
+        return nil
+      end
+
+      return nil if response['incidents'].empty?
+
+      {:pg_acknowledged_by => response['incidents'].first['last_status_change_by']}
+    end
+
   end
 end
 

data/lib/flapjack/redis_pool.rb

@@ -8,7 +8,7 @@ require 'em-synchrony'
 require 'redis/connection/synchrony'
 require 'redis'
 
-# require 'eventmachine/synchrony/connection_pool'
+require 'em-synchrony/connection_pool'
 
 module Flapjack
   class RedisPool < EventMachine::Synchrony::ConnectionPool

data/lib/flapjack/version.rb

@@ -1,5 +1,5 @@
 #!/usr/bin/env ruby
 
 module Flapjack
-  VERSION = "0.6.38"
+  VERSION = "0.6.39"
 end

data/lib/flapjack/web/views/check.haml

@@ -105,7 +105,7 @@
                       %input{:type => 'submit', :value => 'End Now', :class => 'button'}
       - else
         %p No maintenance is scheduled
-      %h4 Add Scheduled Maintenace
+      %h4 Add Scheduled Maintenance
       %form{:action => "/scheduled_maintenances/#{@entity}/#{@check}", :method => "post"}
         %fieldset
           %table

data/spec/lib/flapjack/coordinator_spec.rb

@@ -3,14 +3,171 @@ require 'flapjack/coordinator'
 
 describe Flapjack::Coordinator do
 
-  it "is initialized"
+  let(:fiber) { mock('Fiber') }
 
-  it "starts a suite of services based on config settings"
+  let(:config) {
+    {'redis'          => {},
+     'executive'      => {'enabled' => 'yes'},
+     'email_notifier' => {'enabled' => 'yes'},
+     'web'            => {'enabled' => 'yes'}
+    }
+  }
 
-  it "runs daemonized"
+  before(:each) {
+    # temporary workaround for failing test due to preserved state;
+    # won't be needed soon as this code has been fixed in a branch
+    Flapjack::API.class_variable_set('@@redis', nil)
+    Flapjack::Web.class_variable_set('@@redis', nil)
+  }
 
-  it "runs undaemonized"
+  # leaving actual testing of daemonisation to that class's tests
+  it "daemonizes properly" do
+    fc = Flapjack::Coordinator.new(config)
+    fc.should_receive(:daemonize)
+    fc.should_not_receive(:build_pikelet)
+    fc.should_not_receive(:build_resque_pikelet)
+    fc.should_not_receive(:build_thin_pikelet)
+    fc.start(:daemonize => true, :signals => false)
+  end
 
-  it "stops its services when closing"
+  it "runs undaemonized" do
+    EM.should_receive(:synchrony).and_yield
+
+    fc = Flapjack::Coordinator.new(config)
+    fc.should_receive(:build_pikelet)
+    fc.should_receive(:build_resque_pikelet)
+    fc.should_receive(:build_thin_pikelet)
+    fc.start(:daemonize => false, :signals => false)
+  end
+
+  it "starts after daemonizing" do
+    EM.should_receive(:synchrony).and_yield
+
+    fc = Flapjack::Coordinator.new(config)
+    fc.should_receive(:build_pikelet)
+    fc.should_receive(:build_resque_pikelet)
+    fc.should_receive(:build_thin_pikelet)
+    fc.after_daemonize
+  end
+
+  it "traps system signals and shuts down"
+
+  # TODO whem merged with other changes, this will check pik[:class] instead,
+  # having to create instances of the pikelet classes is messy
+  it "stops its services when closing" do
+    fiber_exec = mock('fiber_exec')
+    fiber_rsq  = mock('fiber_rsq')
+
+    exec  = Flapjack::Executive.new
+    exec.should_receive(:add_shutdown_event)
+    email = EM::Resque::Worker.new('example')
+    email.should_receive(:shutdown)
+    web   = Thin::Server.new('0.0.0.0', 3000, Flapjack::Web, :signals => false)
+    web.should_receive(:stop!)
+
+    redis = mock('redis')
+    redis.should_receive(:quit)
+    Redis.should_receive(:new).and_return(redis)
+
+    fiber.should_receive(:resume)
+    fiber_stop = mock('fiber_stop')
+    fiber_stop.should_receive(:resume)
+    Fiber.should_receive(:new).twice.and_yield.and_return(fiber, fiber_stop)
+
+    fiber_exec.should_receive(:alive?).and_return(true, false)
+    fiber_rsq.should_receive(:alive?).and_return(true, false)
+
+    EM.should_receive(:stop)
+
+    pikelets = [{:fiber => fiber_exec, :instance => exec},
+                {:fiber => fiber_rsq,  :instance => email},
+                {:instance => web}]
+
+    fc = Flapjack::Coordinator.new
+    fc.instance_variable_set('@redis_options', {})
+    fc.instance_variable_set('@pikelets', pikelets)
+    fc.stop
+  end
+
+  it "creates an executive pikelet" do
+    exec = mock('executive')
+    exec.should_receive(:bootstrap)
+    Flapjack::Executive.should_receive(:new).and_return(exec)
+    exec.should_receive(:main)
+
+    fiber.should_receive(:resume)
+    Fiber.should_receive(:new).and_yield.and_return(fiber)
+
+    fc = Flapjack::Coordinator.new
+    fc.send(:build_pikelet, 'executive', {})
+    pikelets = fc.instance_variable_get('@pikelets')
+    pikelets.should_not be_nil
+    pikelets.should be_an(Array)
+    pikelets.should have(1).pikelet
+    pikelets.first.should == {:fiber => fiber, :type => 'executive', :instance => exec}
+  end
+
+  it "handles an exception raised by a jabber pikelet" do
+    jabber = mock('jabber')
+    jabber.should_receive(:bootstrap)
+    Flapjack::Jabber.should_receive(:new).and_return(jabber)
+    jabber.should_receive(:main).and_raise(RuntimeError)
+
+    fiber.should_receive(:resume)
+    Fiber.should_receive(:new).and_yield.and_return(fiber)
+
+    fc = Flapjack::Coordinator.new
+    fc.should_receive(:stop)
+    fc.send(:build_pikelet, 'jabber_gateway', {})
+    pikelets = fc.instance_variable_get('@pikelets')
+    pikelets.should_not be_nil
+    pikelets.should be_an(Array)
+    pikelets.should have(1).pikelet
+    pikelets.first.should == {:fiber => fiber, :type => 'jabber_gateway', :instance => jabber}
+  end
+
+  it "creates a resque worker pikelet" do
+    redis = mock('redis')
+    Flapjack::RedisPool.should_receive(:new).and_return(redis)
+    Resque.should_receive(:redis=).with(redis)
+
+    worker = mock('worker')
+    EM::Resque::Worker.should_receive(:new).and_return(worker)
+    worker.should_receive(:work)
+
+    fiber.should_receive(:resume)
+    Fiber.should_receive(:new).and_yield.and_return(fiber)
+
+    fc = Flapjack::Coordinator.new
+    fc.send(:build_resque_pikelet, 'email_notifier', {})
+    pikelets = fc.instance_variable_get('@pikelets')
+    pikelets.should_not be_nil
+    pikelets.should be_an(Array)
+    pikelets.should have(1).pikelet
+    pikelets.first.should == {:fiber => fiber, :type => 'email_notifier', :instance => worker}
+  end
+
+  it "handles an exception raised by a resque worker pikelet"
+
+  it "creates a thin server pikelet" do
+    redis = mock('redis')
+    Flapjack::RedisPool.should_receive(:new).and_return(redis)
+
+    server = mock('server')
+    server.should_receive(:start)
+    Thin::Server.should_receive(:new).
+      with(/^(?:\d{1,3}\.){3}\d{1,3}$/, an_instance_of(Fixnum), Flapjack::Web, :signals => false).
+      and_return(server)
+
+    fc = Flapjack::Coordinator.new
+    fc.send(:build_thin_pikelet, 'web', {})
+    pikelets = fc.instance_variable_get('@pikelets')
+    pikelets.should_not be_nil
+    pikelets.should be_an(Array)
+    pikelets.should have(1).pikelet
+    pikelets.first.should == {:type => 'web', :instance => server}
+  end
+
+  # NB: exceptions are handled directly by the Thin pikelets
 
 end