upstreamstatus 0.2.4 → 0.2.5

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 77db314deab89fc679fc47c383192fafafab930e
-  data.tar.gz: dc2b44690c28f80881fda94f7a0959dc60198821
+  metadata.gz: 46e8a906db4b57704842d9182ed382e74e6b9604
+  data.tar.gz: 57315acb232238d2932b2860b9627796c6fac02b
 SHA512:
-  metadata.gz: aebccadf9e8857262c831dafd055ecb3f240719a4f6af0b84ade880d5c3a5c4db2e1c2e856f9bcf2a20c2714400132686611f54831e2441080f4a6fae265afcf
-  data.tar.gz: e374ff2d622b4f5ea34a74419d9e5cd0a0b28205b4aeb398d8836c6321eba3b23b3ba9476d5609d722624680a569c7641554c534a0db5d6c6bef836d227487c3
+  metadata.gz: cf7475dacb3d805f689ea558dcedc13c3b430ecf95a824b042e0ac1c129414d74a2886241bbf5d2dc8e52ff0a5b5f62cbf5783b131d35d35a17734cf5a4b4040
+  data.tar.gz: d17cc84a32ee42ac467b1db5f2f3658a1851dd23992d07b75c0120a3d0eda76b35787a888db258914ec9306128e70232c95c75bd6fed89e1144f2109f53b3963

data/lib/upstreamstatus/version.rb

@@ -1,3 +1,3 @@
 class Upstreamstatus
-  VERSION = "0.2.4"
+  VERSION = "0.2.5"
 end

data/lib/upstreamstatus.rb

@@ -1,4 +1,5 @@
 require 'upstreamstatus/version'
+require 'active_support/time'
 require 'unirest'
 require 'yaml'
 require 'ostruct'
@@ -6,6 +7,8 @@ require 'trollop'
 require 'forwardable'
 require 'json'
 require 'sentry-raven'
+require 'pagerduty'
+require 'time'
 
 class Upstreamstatus
   extend Forwardable
@@ -13,14 +16,32 @@ class Upstreamstatus
   def_delegators :@conf,
                  :status_check_url,
                  :sentry_dsn,
-                 :pagerduty_api_key
+                 :pagerduty_api_url,
+                 :pagerduty_service_id,
+                 :pagerduty_api_key,
+                 :pagerduty_rest_api_key
 
   attr_reader :conf
 
   def initialize
     @conf = OpenStruct.new load_conf
 
-    return unless opts[:notify] && sentry_dsn
+    return unless opts[:notify]
+
+    %w(
+      sentry_dsn
+      pagerduty_api_key
+      pagerduty_rest_api_key
+      pagerduty_api_url
+      pagerduty_service_id
+    ).each do |key|
+      fail "Config missing #{key}" unless conf[key]
+    end
+
+    Unirest.default_header 'Authorization',
+                           "Token token=#{pagerduty_rest_api_key}"
+    Unirest.default_header 'Content-type', 'application/json'
+
     Raven.configure do |config|
       config.dsn = sentry_dsn
       config.logger = logger
@@ -36,21 +57,34 @@ class Upstreamstatus
       exit 0
     end
 
+    puts "Detected down hosts:\n"
     print_hosts down_hosts
-    logger.notice "Detected down hosts: #{down_hosts.to_json}"
+    logger.info "Detected down hosts: #{down_hosts.to_json}"
 
     if opts[:notify]
-      puts 'Sending notifications'
       notify(
         'One or more API upstream hosts listed as down',
         JSON.pretty_generate(down_hosts)
       )
     end
     exit 1
+  rescue Interrupt => e
+    puts "Received #{e.class}"
+    exit 99
+  rescue SignalException => e
+    logger.info "Received: #{e.signm} (#{e.signo})"
+    exit 2
+  rescue SystemExit => e
+    exit e.status
+  rescue Exception => e # Need to rescue "Exception" so that Sentry gets it
+    Raven.capture_exception(e) if sentry_dsn
+    logger.fatal e.message
+    logger.fatal e.backtrace.join("\n")
+    raise e
   end
 
   def current_status
-    return fake_servers if opts[:simulate]
+    return fake_response if opts[:simulate]
 
     r = Unirest.get status_check_url
 
@@ -65,34 +99,58 @@ class Upstreamstatus
 
   private
 
-  def fake_servers
-    [
-      {
-        'index' => 0,
-        'upstream' => 'testupstream',
-        'name' => '10.0.0.1 =>8080',
-        'status' => 'up',
-        'rise' => 10_459,
-        'fall' => 0,
-        'type' => 'http',
-        'port' => 0
-      },
-      {
-        'index' => 1,
-        'upstream' => 'testupstream',
-        'name' => '10.0.0.2 =>8080',
-        'status' => 'down',
-        'rise' => 10_029,
-        'fall' => 0,
-        'type' => 'http',
-        'port' => 0
+  def pd_incidents
+    @pd_incidents ||= begin
+      r = Unirest.get(
+        "#{pagerduty_api_url}/incidents",
+        parameters: { service: pagerduty_service_id }
+      )
+      fail "Result: #{r.inspect}" unless (200..299).include?(r.code)
+      r.body['incidents']
+    end
+  end
+
+  def fake_response
+    {
+      'servers' => {
+        'total' => 2,
+        'generation' => 99,
+        'server' => [
+          {
+            'index' => 0,
+            'upstream' => 'testupstream',
+            'name' => '10.0.0.1 =>8080',
+            'status' => 'up',
+            'rise' => 10_459,
+            'fall' => 0,
+            'type' => 'http',
+            'port' => 0
+          },
+          {
+            'index' => 1,
+            'upstream' => 'testupstream',
+            'name' => '10.0.0.2 =>8080',
+            'status' => 'down',
+            'rise' => 10_029,
+            'fall' => 0,
+            'type' => 'http',
+            'port' => 0
+          }
+        ]
       }
-    ]
+    }
   end
 
   def clear_active_alerts!
-    return unless File.exist? '/var/run/active_upstream_alert'
-    File.delete('/var/run/active_upstream_alert')
+    return unless opts[:notify]
+    pd_incidents.each do |i|
+      resolve_incident i['incident_key'] unless i['status'] == 'resolved'
+    end
+  end
+
+  def resolve_incident(incident_key)
+    puts "Resolving incident: #{incident_key}"
+    pagerduty.get_incident(incident_key).resolve
   end
 
   def logger
@@ -104,7 +162,10 @@ class Upstreamstatus
   end
 
   def notify(msg, details)
-    unless active_alert?(msg)
+    if active_alert?(msg)
+      puts "Already an active alert. Not sending anything (message: #{msg})"
+    else
+      puts "Notifying PagerDuty (message: #{msg})"
       pd = pagerduty.trigger(
         msg,
         client: ENV['hostname'],
@@ -124,14 +185,13 @@ class Upstreamstatus
         }.to_json
       )
     end
-
-    Raven.capture_exception(e) if sentry_dsn
   end
 
   def active_alert?(msg)
-    File.exist?('/var/run/active_upstream_alert') &&
-      (Time.now - File.ctime('/var/run/active_upstream_alert')) < 3600 &&
-      JSON.parse(File.read('/var/run/active_upstream_alert'))['msg'] == msg
+    pd_incidents.find do |incident|
+      incident['status'] != 'resolved' &&
+        incident['trigger_summary_data']['description'] == msg
+    end
   end
 
   def opts

data/upstreamstatus.gemspec

@@ -30,6 +30,8 @@ Gem::Specification.new do |spec|
   spec.add_dependency 'unirest', '~> 1.1'
   spec.add_dependency 'trollop', '~> 2.1'
   spec.add_dependency 'sentry-raven'
+  spec.add_dependency 'pagerduty'
+  spec.add_dependency 'activesupport'
 
   spec.add_development_dependency 'bundler', '~> 1.10'
   spec.add_development_dependency 'rake', '~> 10.0'

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: upstreamstatus
 version: !ruby/object:Gem::Version
-  version: 0.2.4
+  version: 0.2.5
 platform: ruby
 authors:
 - Eric Herot
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2015-11-06 00:00:00.000000000 Z
+date: 2015-11-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: unirest
@@ -52,6 +52,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: pagerduty
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: bundler
   requirement: !ruby/object:Gem::Requirement
@@ -156,4 +184,3 @@ signing_key:
 specification_version: 4
 summary: Parse the output of the Nginx Upstream Check plugin
 test_files: []
-has_rdoc: